From 455c781c6d1e1161f66e275299cf06064a0ffde2 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Tue, 19 Nov 2024 15:28:26 +0100 Subject: [PATCH] [LLM tasks] Add product documentation retrieval task (#194379) ## Summary Close https://github.com/elastic/kibana/issues/193473 Close https://github.com/elastic/kibana/issues/193474 This PR utilize the documentation packages that are build via the tool introduced by https://github.com/elastic/kibana/pull/193847, allowing to install them in Kibana and expose documentation retrieval as an LLM task that AI assistants (or other consumers) can call. Users can now decide to install the Elastic documentation from the assistant's config screen, which will expose a new tool for the assistant, `retrieve_documentation` (only implemented for the o11y assistant in the current PR, shall be done for security as a follow up). For more information, please refer to the self-review. ## General architecture Screenshot 2024-10-17 at 09 22 32 ## What this PR does Adds two plugin: - `productDocBase`: contains all the logic related to product documentation installation, status, and search. This is meant to be a "low level" components only responsible for this specific part. - `llmTasks`: an higher level plugin that will contain various LLM tasks to be used by assistants and genAI consumers. The intent is not to have a single place to put all llm tasks, but more to have a default place where we can introduce new tasks from. (fwiw, the `nlToEsql` task will probably be moved to that plugin). - Add a `retrieve_documentation` tool registration for the o11y assistant - Add a component on the o11y assistant configuration page to install the product doc (wiring the feature to the o11y assistant was done for testing purposes mostly, any addition / changes / enhancement should be done by the owning team - either in this PR or as a follow-up) ## What is NOT included in this PR: - Wire product base feature to the security assistant (should be done by the owning team as a follow-up) - installation - utilization as tool - FTR tests: this is somewhat blocked by the same things we need to figure out for https://github.com/elastic/kibana-team/issues/1271 ## Screenshots ### Installation from o11y assistant configuration page Screenshot 2024-10-17 at 09 41 24 ### Example of output #### Without product documentation installed Screenshot 2024-10-10 at 09 59 41 #### With product documentation installed Screenshot 2024-10-10 at 09 55 38 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Alex Szabo Co-authored-by: Matthias Wilhelm Co-authored-by: Elastic Machine --- .github/CODEOWNERS | 3 + docs/developer/plugin-list.asciidoc | 8 + docs/user/security/audit-logging.asciidoc | 9 + package.json | 3 + .../current_fields.json | 7 + .../current_mappings.json | 20 ++ packages/kbn-optimizer/limits.yml | 1 + .../check_registered_types.test.ts | 1 + .../group3/type_registrations.test.ts | 1 + tsconfig.base.json | 6 + .../product-doc-artifact-builder/README.md | 48 +++- .../src/artifact/manifest.ts | 8 +- .../src/artifact/mappings.ts | 5 +- .../src/artifact/product_name.ts | 35 ++- .../src/build_artifacts.ts | 8 +- .../src/command.ts | 8 +- .../src/tasks/create_artifact.ts | 4 +- .../src/tasks/create_chunk_files.ts | 2 +- .../src/tasks/create_index.ts | 5 +- .../src/tasks/extract_documentation.ts | 10 +- .../src/tasks/index.ts | 2 +- .../src/tasks/process_documents.ts | 59 ++++ .../product-doc-artifact-builder/src/types.ts | 4 +- .../tsconfig.json | 1 + .../ai-infra/product-doc-common/README.md | 3 + .../ai-infra/product-doc-common/index.ts | 17 ++ .../product-doc-common/jest.config.js | 12 + .../ai-infra/product-doc-common/kibana.jsonc | 5 + .../ai-infra/product-doc-common/package.json | 6 + .../product-doc-common/src/artifact.test.ts | 64 +++++ .../product-doc-common/src/artifact.ts | 39 +++ .../src/artifact_content.test.ts | 23 ++ .../src/artifact_content.ts | 12 + .../product-doc-common/src/documents.ts | 31 +++ .../product-doc-common/src/indices.ts | 15 ++ .../src/manifest.ts} | 14 +- .../product-doc-common/src/product.ts | 15 ++ .../ai-infra/product-doc-common/tsconfig.json | 17 ++ x-pack/plugins/ai_infra/llm_tasks/README.md | 45 ++++ .../plugins/ai_infra/llm_tasks/jest.config.js | 19 ++ .../plugins/ai_infra/llm_tasks/kibana.jsonc | 15 ++ .../ai_infra/llm_tasks/server/config.ts | 18 ++ .../ai_infra/llm_tasks/server/index.ts | 28 ++ .../ai_infra/llm_tasks/server/plugin.ts | 56 ++++ .../ai_infra/llm_tasks/server/tasks/index.ts | 8 + .../tasks/retrieve_documentation/index.ts | 13 + .../retrieve_documentation.test.ts | 182 +++++++++++++ .../retrieve_documentation.ts | 88 ++++++ .../summarize_document.ts | 67 +++++ .../tasks/retrieve_documentation/types.ts | 72 +++++ .../ai_infra/llm_tasks/server/types.ts | 42 +++ .../llm_tasks/server/utils/tokens.test.ts | 27 ++ .../ai_infra/llm_tasks/server/utils/tokens.ts | 21 ++ .../plugins/ai_infra/llm_tasks/tsconfig.json | 27 ++ .../ai_infra/product_doc_base/README.md | 3 + .../product_doc_base/common/consts.ts | 14 + .../common/http_api/installation.ts | 26 ++ .../product_doc_base/common/install_status.ts | 28 ++ .../ai_infra/product_doc_base/jest.config.js | 23 ++ .../ai_infra/product_doc_base/kibana.jsonc | 15 ++ .../ai_infra/product_doc_base/public/index.ts | 26 ++ .../product_doc_base/public/plugin.tsx | 51 ++++ .../public/services/installation/index.ts | 9 + .../installation/installation_service.test.ts | 79 ++++++ .../installation/installation_service.ts | 40 +++ .../public/services/installation/types.ts | 18 ++ .../ai_infra/product_doc_base/public/types.ts | 22 ++ .../product_doc_base/server/config.ts | 22 ++ .../ai_infra/product_doc_base/server/index.ts | 29 ++ .../product_doc_base/server/plugin.test.ts | 96 +++++++ .../product_doc_base/server/plugin.ts | 133 +++++++++ .../product_doc_base/server/routes/index.ts | 20 ++ .../server/routes/installation.ts | 115 ++++++++ .../server/saved_objects/index.ts | 11 + .../saved_objects/product_doc_install.ts | 46 ++++ .../services/doc_install_status/index.ts | 8 + .../model_conversion.test.ts | 44 +++ .../doc_install_status/model_conversion.ts | 26 ++ .../product_doc_install_service.test.ts | 65 +++++ .../product_doc_install_service.ts | 89 ++++++ .../doc_install_status/service.mock.ts | 24 ++ .../services/doc_manager/check_license.ts | 13 + .../services/doc_manager/doc_manager.test.ts | 247 +++++++++++++++++ .../services/doc_manager/doc_manager.ts | 204 ++++++++++++++ .../server/services/doc_manager/index.ts | 15 ++ .../server/services/doc_manager/types.ts | 98 +++++++ .../endpoint_manager.test.ts | 58 ++++ .../inference_endpoint/endpoint_manager.ts | 41 +++ .../services/inference_endpoint/index.ts | 8 + .../inference_endpoint/service.mock.ts | 20 ++ .../utils/get_model_install_status.ts | 34 +++ .../inference_endpoint/utils/index.ts | 10 + .../inference_endpoint/utils/install_elser.ts | 35 +++ .../utils/wait_until_model_deployed.ts | 39 +++ .../services/package_installer/index.ts | 8 + .../package_installer.test.mocks.ts | 36 +++ .../package_installer.test.ts | 255 ++++++++++++++++++ .../package_installer/package_installer.ts | 218 +++++++++++++++ .../steps/create_index.test.ts | 84 ++++++ .../package_installer/steps/create_index.ts | 50 ++++ .../steps/fetch_artifact_versions.test.ts | 129 +++++++++ .../steps/fetch_artifact_versions.ts | 59 ++++ .../services/package_installer/steps/index.ts | 11 + .../steps/populate_index.test.ts | 109 ++++++++ .../package_installer/steps/populate_index.ts | 84 ++++++ .../steps/validate_artifact_archive.test.ts | 73 +++++ .../steps/validate_artifact_archive.ts | 24 ++ .../utils/archive_accessors.test.ts | 78 ++++++ .../utils/archive_accessors.ts | 33 +++ .../package_installer/utils/download.ts | 23 ++ .../services/package_installer/utils/index.ts | 10 + .../package_installer/utils/semver.test.ts | 29 ++ .../package_installer/utils/semver.ts | 27 ++ .../utils/test_data/test_archive_1.zip | Bin 0 -> 800 bytes .../utils/zip_archive.test.ts | 43 +++ .../package_installer/utils/zip_archive.ts | 91 +++++++ .../server/services/search/index.ts | 9 + .../server/services/search/perform_search.ts} | 28 +- .../services/search/search_service.test.ts | 51 ++++ .../server/services/search/search_service.ts | 37 +++ .../server/services/search/types.ts | 27 ++ .../get_indices_for_product_names.test.ts | 22 ++ .../utils/get_indices_for_product_names.ts | 21 ++ .../server/services/search/utils/index.ts | 9 + .../services/search/utils/map_result.test.ts | 46 ++++ .../services/search/utils/map_result.ts | 19 ++ .../server/tasks/ensure_up_to_date.ts | 70 +++++ .../product_doc_base/server/tasks/index.ts | 29 ++ .../server/tasks/install_all.ts | 70 +++++ .../server/tasks/uninstall_all.ts | 70 +++++ .../product_doc_base/server/tasks/utils.ts | 69 +++++ .../ai_infra/product_doc_base/server/types.ts | 44 +++ .../ai_infra/product_doc_base/tsconfig.json | 29 ++ .../server/plugin.ts | 2 +- .../kibana.jsonc | 3 +- .../server/functions/documentation.ts | 82 ++++++ .../server/functions/index.ts | 2 + .../server/types.ts | 2 + .../tsconfig.json | 2 + .../kibana.jsonc | 3 +- .../public/constants.ts | 3 + .../hooks/use_get_product_doc_status.ts | 32 +++ .../public/hooks/use_install_product_doc.ts | 57 ++++ .../public/hooks/use_uninstall_product_doc.ts | 57 ++++ .../public/plugin.ts | 2 + .../settings_tab/product_doc_entry.tsx | 171 ++++++++++++ .../components/settings_tab/settings_tab.tsx | 3 + .../tsconfig.json | 3 +- .../check_registered_task_types.ts | 3 + yarn.lock | 12 + 150 files changed, 5662 insertions(+), 64 deletions(-) create mode 100644 x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/README.md create mode 100644 x-pack/packages/ai-infra/product-doc-common/index.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/jest.config.js create mode 100644 x-pack/packages/ai-infra/product-doc-common/kibana.jsonc create mode 100644 x-pack/packages/ai-infra/product-doc-common/package.json create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/documents.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/indices.ts rename x-pack/packages/ai-infra/{product-doc-artifact-builder/src/artifact/artifact_name.ts => product-doc-common/src/manifest.ts} (59%) create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/product.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/tsconfig.json create mode 100644 x-pack/plugins/ai_infra/llm_tasks/README.md create mode 100644 x-pack/plugins/ai_infra/llm_tasks/jest.config.js create mode 100644 x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/config.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/types.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/tsconfig.json create mode 100644 x-pack/plugins/ai_infra/product_doc_base/README.md create mode 100644 x-pack/plugins/ai_infra/product_doc_base/common/consts.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/jest.config.js create mode 100644 x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/config.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts rename x-pack/{packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts => plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts} (78%) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/tsconfig.json create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0894fbff896a..d7e2cf7fda61 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -768,6 +768,7 @@ x-pack/examples/triggers_actions_ui_example @elastic/response-ops x-pack/examples/ui_actions_enhanced_examples @elastic/appex-sharedux x-pack/packages/ai-infra/inference-common @elastic/appex-ai-infra x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra +x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared @elastic/kibana-management x-pack/packages/index-management/index_management_shared_types @elastic/kibana-management x-pack/packages/kbn-ai-assistant @elastic/search-kibana @@ -857,6 +858,8 @@ x-pack/packages/security/role_management_model @elastic/kibana-security x-pack/packages/security/ui_components @elastic/kibana-security x-pack/performance @elastic/appex-qa x-pack/plugins/actions @elastic/response-ops +x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra +x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra x-pack/plugins/aiops @elastic/ml-ui x-pack/plugins/alerting @elastic/response-ops x-pack/plugins/banners @elastic/appex-sharedux diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 71ab26400f49..ea3186357611 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana. using the CURL scripts in the scripts folder. +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks] +|This plugin contains various LLM tasks. + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess] |Exposes services to access logs data. @@ -767,6 +771,10 @@ Elastic. |This plugin helps users learn how to use the Painless scripting language. +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase] +|This plugin contains the product documentation base service. + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling] |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views. diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc index 1ac40bcc7764..ef12f4303c1b 100644 --- a/docs/user/security/audit-logging.asciidoc +++ b/docs/user/security/audit-logging.asciidoc @@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Creating trained model. | `failure` | Failed to create trained model. +.1+| `product_documentation_create` +| `unknown` | User requested to install the product documentation for use in AI Assistants. + 3+a| ====== Type: change @@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Updating trained model deployment. | `failure` | Failed to update trained model deployment. +.1+| `product_documentation_update` +| `unknown` | User requested to update the product documentation for use in AI Assistants. + 3+a| ====== Type: deletion @@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Deleting trained model. | `failure` | Failed to delete trained model. +.1+| `product_documentation_delete` +| `unknown` | User requested to delete the product documentation for use in AI Assistants. + 3+a| ====== Type: access diff --git a/package.json b/package.json index ebbf26f25002..b67d4b90fdf9 100644 --- a/package.json +++ b/package.json @@ -617,6 +617,7 @@ "@kbn/licensing-plugin": "link:x-pack/plugins/licensing", "@kbn/links-plugin": "link:src/plugins/links", "@kbn/lists-plugin": "link:x-pack/plugins/lists", + "@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks", "@kbn/locator-examples-plugin": "link:examples/locator_examples", "@kbn/locator-explorer-plugin": "link:examples/locator_explorer", "@kbn/logging": "link:packages/kbn-logging", @@ -721,6 +722,8 @@ "@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel", "@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing", "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util", + "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base", + "@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common", "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access", "@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling", "@kbn/profiling-utils": "link:packages/kbn-profiling-utils", diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index a5642cee1095..020b9a97753b 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -855,6 +855,13 @@ "policy-settings-protection-updates-note": [ "note" ], + "product-doc-install-status": [ + "index_name", + "installation_status", + "last_installation_date", + "product_name", + "product_version" + ], "query": [ "description", "title", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 61f680509c13..2409b7578da8 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2841,6 +2841,26 @@ } } }, + "product-doc-install-status": { + "dynamic": false, + "properties": { + "index_name": { + "type": "keyword" + }, + "installation_status": { + "type": "keyword" + }, + "last_installation_date": { + "type": "date" + }, + "product_name": { + "type": "keyword" + }, + "product_version": { + "type": "keyword" + } + } + }, "query": { "dynamic": false, "properties": { diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 58424700d9bf..32a7bc827907 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -124,6 +124,7 @@ pageLoadAssetSize: painlessLab: 179748 presentationPanel: 55463 presentationUtil: 58834 + productDocBase: 22500 profiling: 36694 remoteClusters: 51327 reporting: 58600 diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 28a1e8e1eb53..0f186fba94b5 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () => "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75", "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4", "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352", + "product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c", "query": "501bece68f26fe561286a488eabb1a8ab12f1137", "risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb", "rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index ba06073e454a..3ceba522d08c 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -115,6 +115,7 @@ const previouslyRegisteredTypes = [ 'osquery-usage-metric', 'osquery-manager-usage-metric', 'policy-settings-protection-updates-note', + 'product-doc-install-status', 'query', 'rules-settings', 'sample-data-telemetry', diff --git a/tsconfig.base.json b/tsconfig.base.json index 26fe060916a9..3e1d80208f5b 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1148,6 +1148,8 @@ "@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"], "@kbn/lists-plugin": ["x-pack/plugins/lists"], "@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"], + "@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"], + "@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"], "@kbn/locator-examples-plugin": ["examples/locator_examples"], "@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"], "@kbn/locator-explorer-plugin": ["examples/locator_explorer"], @@ -1390,6 +1392,10 @@ "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"], "@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"], "@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"], + "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"], + "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"], + "@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"], + "@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"], "@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"], "@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"], "@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"], diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md index eb64d53b5b8f..49949def3e5e 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md @@ -1,3 +1,49 @@ # @kbn/product-doc-artifact-builder -Script to build the knowledge base artifacts +Script to build the knowledge base artifacts. + +## How to run + +``` +node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product} +``` + +### parameters + +#### `stack-version`: + +the stack version to generate the artifacts for. + +#### `product-name`: + +(multi-value) the list of products to generate artifacts for. + +possible values: +- "kibana" +- "elasticsearch" +- "observability" +- "security" + +#### `target-folder`: + +The folder to generate the artifacts in. + +Defaults to `{REPO_ROOT}/build-kb-artifacts`. + +#### `build-folder`: + +The folder to use for temporary files. + +Defaults to `{REPO_ROOT}/build/temp-kb-artifacts` + +#### Cluster infos + +- params for the source cluster: +`sourceClusterUrl` / env.KIBANA_SOURCE_CLUSTER_URL +`sourceClusterUsername` / env.KIBANA_SOURCE_CLUSTER_USERNAME +`sourceClusterPassword` / env.KIBANA_SOURCE_CLUSTER_PASSWORD + +- params for the embedding cluster: +`embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL +`embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME +`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD \ No newline at end of file diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts index cbebcdc22981..a8aa927c5ef1 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts @@ -5,17 +5,13 @@ * 2.0. */ -export interface ArtifactManifest { - formatVersion: string; - productName: string; - productVersion: string; -} +import type { ArtifactManifest, ProductName } from '@kbn/product-doc-common'; export const getArtifactManifest = ({ productName, stackVersion, }: { - productName: string; + productName: ProductName; stackVersion: string; }): ArtifactManifest => { return { diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts index ae84ae60616a..979845ec3184 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts @@ -21,10 +21,7 @@ export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMappi slug: { type: 'keyword' }, url: { type: 'keyword' }, version: { type: 'version' }, - ai_subtitle: { - type: 'semantic_text', - inference_id: inferenceEndpoint, - }, + ai_subtitle: { type: 'text' }, ai_summary: { type: 'semantic_text', inference_id: inferenceEndpoint, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts index cfcc141323f4..e4ca33849a52 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts @@ -5,7 +5,34 @@ * 2.0. */ -/** - * The allowed product names, as found in the source's cluster - */ -export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability']; +import type { ProductName } from '@kbn/product-doc-common'; + +const productNameToSourceNamesMap: Record = { + kibana: ['Kibana'], + elasticsearch: ['Elasticsearch'], + security: ['Security'], + observability: ['Observability'], +}; + +const sourceNameToProductName = Object.entries(productNameToSourceNamesMap).reduce< + Record +>((map, [productName, sourceNames]) => { + sourceNames.forEach((sourceName) => { + map[sourceName] = productName as ProductName; + }); + return map; +}, {}); + +export const getSourceNamesFromProductName = (productName: ProductName): string[] => { + if (!productNameToSourceNamesMap[productName]) { + throw new Error(`Unknown product name: ${productName}`); + } + return productNameToSourceNamesMap[productName]; +}; + +export const getProductNameFromSource = (source: string): ProductName => { + if (!sourceNameToProductName[source]) { + throw new Error(`Unknown source name: ${source}`); + } + return sourceNameToProductName[source]; +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts index bbde3310f8e3..551f58bc6830 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts @@ -8,6 +8,7 @@ import Path from 'path'; import { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; +import type { ProductName } from '@kbn/product-doc-common'; import { // checkConnectivity, createTargetIndex, @@ -18,6 +19,7 @@ import { createArtifact, cleanupFolders, deleteIndex, + processDocuments, } from './tasks'; import type { TaskConfig } from './types'; @@ -93,7 +95,7 @@ const buildArtifact = async ({ sourceClient, log, }: { - productName: string; + productName: ProductName; stackVersion: string; buildFolder: string; targetFolder: string; @@ -105,7 +107,7 @@ const buildArtifact = async ({ const targetIndex = getTargetIndexName({ productName, stackVersion }); - const documents = await extractDocumentation({ + let documents = await extractDocumentation({ client: sourceClient, index: 'search-docs-1', log, @@ -113,6 +115,8 @@ const buildArtifact = async ({ stackVersion, }); + documents = await processDocuments({ documents, log }); + await createTargetIndex({ client: embeddingClient, indexName: targetIndex, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts index 49af1d158db8..e8d0d9486e33 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts @@ -6,19 +6,19 @@ */ import Path from 'path'; -import { REPO_ROOT } from '@kbn/repo-info'; import yargs from 'yargs'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { DocumentationProduct } from '@kbn/product-doc-common'; import type { TaskConfig } from './types'; import { buildArtifacts } from './build_artifacts'; -import { sourceProductNames } from './artifact/product_name'; function options(y: yargs.Argv) { return y .option('productName', { describe: 'name of products to generate documentation for', array: true, - choices: sourceProductNames, - default: ['Kibana'], + choices: Object.values(DocumentationProduct), + default: [DocumentationProduct.kibana], }) .option('stackVersion', { describe: 'The stack version to generate documentation for', diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts index 343099876585..056887a41a4d 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts @@ -8,9 +8,9 @@ import Path from 'path'; import AdmZip from 'adm-zip'; import type { ToolingLog } from '@kbn/tooling-log'; +import { getArtifactName, type ProductName } from '@kbn/product-doc-common'; import { getArtifactMappings } from '../artifact/mappings'; import { getArtifactManifest } from '../artifact/manifest'; -import { getArtifactName } from '../artifact/artifact_name'; export const createArtifact = async ({ productName, @@ -21,7 +21,7 @@ export const createArtifact = async ({ }: { buildFolder: string; targetFolder: string; - productName: string; + productName: ProductName; stackVersion: string; log: ToolingLog; }) => { diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts index 8b0e7323c288..73cf8f010922 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts @@ -10,7 +10,7 @@ import Fs from 'fs/promises'; import type { Client } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; -const fileSizeLimit = 250_000; +const fileSizeLimit = 500_000; export const createChunkFiles = async ({ index, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts index e4f24725883a..d26ffc980f3a 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts @@ -21,10 +21,7 @@ const mappings: MappingTypeMapping = { slug: { type: 'keyword' }, url: { type: 'keyword' }, version: { type: 'version' }, - ai_subtitle: { - type: 'semantic_text', - inference_id: 'kibana-elser2', - }, + ai_subtitle: { type: 'text' }, ai_summary: { type: 'semantic_text', inference_id: 'kibana-elser2', diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts index f1dd051394bb..6aa8bb49b0cf 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts @@ -8,6 +8,8 @@ import type { Client } from '@elastic/elasticsearch'; import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import type { ToolingLog } from '@kbn/tooling-log'; +import type { ProductName } from '@kbn/product-doc-common'; +import { getSourceNamesFromProductName, getProductNameFromSource } from '../artifact/product_name'; /** the list of fields to import from the source cluster */ const fields = [ @@ -27,7 +29,7 @@ const fields = [ export interface ExtractedDocument { content_title: string; content_body: string; - product_name: string; + product_name: ProductName; root_type: string; slug: string; url: string; @@ -43,7 +45,7 @@ const convertHit = (hit: SearchHit): ExtractedDocument => { return { content_title: source.content_title, content_body: source.content_body, - product_name: source.product_name, + product_name: getProductNameFromSource(source.product_name), root_type: 'documentation', slug: source.slug, url: source.url, @@ -65,7 +67,7 @@ export const extractDocumentation = async ({ client: Client; index: string; stackVersion: string; - productName: string; + productName: ProductName; log: ToolingLog; }) => { log.info(`Starting to extract documents from source cluster`); @@ -76,7 +78,7 @@ export const extractDocumentation = async ({ query: { bool: { must: [ - { term: { product_name: productName } }, + { terms: { product_name: getSourceNamesFromProductName(productName) } }, { term: { version: stackVersion } }, { exists: { field: 'ai_fields.ai_summary' } }, ], diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts index 0c6343136232..ec94e4c135c1 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts @@ -10,8 +10,8 @@ export { indexDocuments } from './index_documents'; export { createTargetIndex } from './create_index'; export { installElser } from './install_elser'; export { createChunkFiles } from './create_chunk_files'; -export { performSemanticSearch } from './perform_semantic_search'; export { checkConnectivity } from './check_connectivity'; export { createArtifact } from './create_artifact'; export { cleanupFolders } from './cleanup_folders'; export { deleteIndex } from './delete_index'; +export { processDocuments } from './process_documents'; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts new file mode 100644 index 000000000000..69141ca167ab --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniqBy } from 'lodash'; +import { encode } from 'gpt-tokenizer'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { ExtractedDocument } from './extract_documentation'; + +export const processDocuments = async ({ + documents, + log, +}: { + documents: ExtractedDocument[]; + log: ToolingLog; +}): Promise => { + log.info('Starting processing documents.'); + const initialCount = documents.length; + documents = removeDuplicates(documents); + const noDupCount = documents.length; + log.info(`Removed ${initialCount - noDupCount} duplicates`); + documents.forEach(processDocument); + documents = filterEmptyDocs(documents); + log.info(`Removed ${noDupCount - documents.length} empty documents`); + log.info('Done processing documents.'); + return documents; +}; + +const removeDuplicates = (documents: ExtractedDocument[]): ExtractedDocument[] => { + return uniqBy(documents, (doc) => doc.slug); +}; + +/** + * Filter "this content has moved" or "deleted pages" type of documents, just based on token count. + */ +const filterEmptyDocs = (documents: ExtractedDocument[]): ExtractedDocument[] => { + return documents.filter((doc) => { + const tokenCount = encode(doc.content_body).length; + if (tokenCount < 100) { + return false; + } + return true; + }); +}; + +const processDocument = (document: ExtractedDocument) => { + document.content_body = document.content_body + // remove those "edit" button text that got embedded into titles. + .replaceAll(/([a-zA-Z])edit\n/g, (match) => { + return `${match[0]}\n`; + }) + // limit to 2 consecutive carriage return + .replaceAll(/\n\n+/g, '\n\n'); + + return document; +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts index d2acfb577450..1eb4a4348d21 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts @@ -5,8 +5,10 @@ * 2.0. */ +import type { ProductName } from '@kbn/product-doc-common'; + export interface TaskConfig { - productNames: string[]; + productNames: ProductName[]; stackVersion: string; buildFolder: string; targetFolder: string; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json b/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json index 508d4c715d0a..68ff27852c4d 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json @@ -16,5 +16,6 @@ "kbn_references": [ "@kbn/tooling-log", "@kbn/repo-info", + "@kbn/product-doc-common", ] } diff --git a/x-pack/packages/ai-infra/product-doc-common/README.md b/x-pack/packages/ai-infra/product-doc-common/README.md new file mode 100644 index 000000000000..ff20c0e0fd0e --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/README.md @@ -0,0 +1,3 @@ +# @kbn/product-doc-common + +Common types and utilities for the product documentation feature. diff --git a/x-pack/packages/ai-infra/product-doc-common/index.ts b/x-pack/packages/ai-infra/product-doc-common/index.ts new file mode 100644 index 000000000000..1a9673713899 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getArtifactName, parseArtifactName } from './src/artifact'; +export { type ArtifactManifest } from './src/manifest'; +export { DocumentationProduct, type ProductName } from './src/product'; +export { isArtifactContentFilePath } from './src/artifact_content'; +export { + productDocIndexPrefix, + productDocIndexPattern, + getProductDocIndexName, +} from './src/indices'; +export type { ProductDocumentationAttributes } from './src/documents'; diff --git a/x-pack/packages/ai-infra/product-doc-common/jest.config.js b/x-pack/packages/ai-infra/product-doc-common/jest.config.js new file mode 100644 index 000000000000..e6cae43806c8 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../..', + roots: ['/x-pack/packages/ai-infra/product-doc-common'], +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc b/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc new file mode 100644 index 000000000000..16336c1fc8e2 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/product-doc-common", + "owner": "@elastic/appex-ai-infra" +} diff --git a/x-pack/packages/ai-infra/product-doc-common/package.json b/x-pack/packages/ai-infra/product-doc-common/package.json new file mode 100644 index 000000000000..839d411a2efb --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/product-doc-common", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} \ No newline at end of file diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts new file mode 100644 index 000000000000..2b6362dbf4aa --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getArtifactName, parseArtifactName } from './artifact'; + +describe('getArtifactName', () => { + it('builds the name based on the provided product name and version', () => { + expect( + getArtifactName({ + productName: 'kibana', + productVersion: '8.16', + }) + ).toEqual('kb-product-doc-kibana-8.16.zip'); + }); + + it('excludes the extension when excludeExtension is true', () => { + expect( + getArtifactName({ + productName: 'elasticsearch', + productVersion: '8.17', + excludeExtension: true, + }) + ).toEqual('kb-product-doc-elasticsearch-8.17'); + }); + + it('generates a lowercase name', () => { + expect( + getArtifactName({ + // @ts-expect-error testing + productName: 'ElasticSearch', + productVersion: '8.17', + excludeExtension: true, + }) + ).toEqual('kb-product-doc-elasticsearch-8.17'); + }); +}); + +describe('parseArtifactName', () => { + it('parses an artifact name with extension', () => { + expect(parseArtifactName('kb-product-doc-kibana-8.16.zip')).toEqual({ + productName: 'kibana', + productVersion: '8.16', + }); + }); + + it('parses an artifact name without extension', () => { + expect(parseArtifactName('kb-product-doc-security-8.17')).toEqual({ + productName: 'security', + productVersion: '8.17', + }); + }); + + it('returns undefined if the provided string does not match the artifact name pattern', () => { + expect(parseArtifactName('some-wrong-name')).toEqual(undefined); + }); + + it('returns undefined if the provided string is not strictly lowercase', () => { + expect(parseArtifactName('kb-product-doc-Security-8.17')).toEqual(undefined); + }); +}); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts new file mode 100644 index 000000000000..1a6745abd733 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type ProductName, DocumentationProduct } from './product'; + +// kb-product-doc-elasticsearch-8.15.zip +const artifactNameRegexp = /^kb-product-doc-([a-z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; +const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); + +export const getArtifactName = ({ + productName, + productVersion, + excludeExtension = false, +}: { + productName: ProductName; + productVersion: string; + excludeExtension?: boolean; +}): string => { + const ext = excludeExtension ? '' : '.zip'; + return `kb-product-doc-${productName}-${productVersion}${ext}`.toLowerCase(); +}; + +export const parseArtifactName = (artifactName: string) => { + const match = artifactNameRegexp.exec(artifactName); + if (match) { + const productName = match[1].toLowerCase() as ProductName; + const productVersion = match[2].toLowerCase(); + if (allowedProductNames.includes(productName)) { + return { + productName, + productVersion, + }; + } + } +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts new file mode 100644 index 000000000000..3f97aaf94f88 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArtifactContentFilePath } from './artifact_content'; + +describe('isArtifactContentFilePath', () => { + it('returns true for filenames matching the pattern', () => { + expect(isArtifactContentFilePath('content/content-0.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-007.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-9042.ndjson')).toEqual(true); + }); + + it('returns false for filenames not matching the pattern', () => { + expect(isArtifactContentFilePath('content-0.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content/content-0')).toEqual(false); + expect(isArtifactContentFilePath('content/content.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content/content-9042.json')).toEqual(false); + }); +}); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts new file mode 100644 index 000000000000..757e6664bb58 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; + +export const isArtifactContentFilePath = (path: string): boolean => { + return contentFileRegexp.test(path); +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts new file mode 100644 index 000000000000..ef81b3d6411c --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from './product'; + +// don't need to define the other props +interface SemanticTextField { + text: string; +} + +interface SemanticTextArrayField { + text: string[]; +} + +export interface ProductDocumentationAttributes { + content_title: string; + content_body: SemanticTextField; + product_name: ProductName; + root_type: string; + slug: string; + url: string; + version: string; + ai_subtitle: string; + ai_summary: SemanticTextField; + ai_questions_answered: SemanticTextArrayField; + ai_tags: string[]; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/src/indices.ts b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts new file mode 100644 index 000000000000..b48cacf79fd2 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from './product'; + +export const productDocIndexPrefix = '.kibana-ai-product-doc'; +export const productDocIndexPattern = `${productDocIndexPrefix}-*`; + +export const getProductDocIndexName = (productName: ProductName): string => { + return `${productDocIndexPrefix}-${productName.toLowerCase()}`; +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts similarity index 59% rename from x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts rename to x-pack/packages/ai-infra/product-doc-common/src/manifest.ts index 678b17088c7b..6c246cf58fd5 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts @@ -5,12 +5,10 @@ * 2.0. */ -export const getArtifactName = ({ - productName, - productVersion, -}: { - productName: string; +import type { ProductName } from './product'; + +export interface ArtifactManifest { + formatVersion: string; + productName: ProductName; productVersion: string; -}): string => { - return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase(); -}; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/src/product.ts b/x-pack/packages/ai-infra/product-doc-common/src/product.ts new file mode 100644 index 000000000000..417033f5083e --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/product.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum DocumentationProduct { + kibana = 'kibana', + elasticsearch = 'elasticsearch', + observability = 'observability', + security = 'security', +} + +export type ProductName = keyof typeof DocumentationProduct; diff --git a/x-pack/packages/ai-infra/product-doc-common/tsconfig.json b/x-pack/packages/ai-infra/product-doc-common/tsconfig.json new file mode 100644 index 000000000000..0d78dace105e --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [] +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/README.md b/x-pack/plugins/ai_infra/llm_tasks/README.md new file mode 100644 index 000000000000..e019d456cd65 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/README.md @@ -0,0 +1,45 @@ +# LLM Tasks plugin + +This plugin contains various LLM tasks. + +## Retrieve documentation + +This task allows to retrieve documents from our Elastic product documentation. + +The task depends on the `product-doc-base` plugin, as this dependency is used +to install and manage the product documentation. + +### Checking if the task is available + +A `retrieveDocumentationAvailable` API is exposed from the start contract, that +should be used to assert that the `retrieve_doc` task can be used in the current +context. + +That API receive the inbound request as parameter. + +Example: +```ts +if (await llmTasksStart.retrieveDocumentationAvailable({ request })) { + // task is available +} else { + // task is not available +} +``` + +### Executing the task + +The task is executed as an API of the plugin's start contract, and can be invoked +as any other lifecycle API would. + +Example: +```ts +const result = await llmTasksStart.retrieveDocumentation({ + searchTerm: "How to create a space in Kibana?", + request, + connectorId: 'my-connector-id', +}); + +const { success, documents } = result; +``` + +The exhaustive list of options for the task is available on the `RetrieveDocumentationParams` type's TS doc. diff --git a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js new file mode 100644 index 000000000000..2a6206d4304b --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: ['/x-pack/plugins/ai_infra/llm_tasks/server'], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/ai_infra/llm_tasks/{public,server,common}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc new file mode 100644 index 000000000000..1ef211d01210 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc @@ -0,0 +1,15 @@ +{ + "type": "plugin", + "id": "@kbn/llm-tasks-plugin", + "owner": "@elastic/appex-ai-infra", + "plugin": { + "id": "llmTasks", + "server": true, + "browser": false, + "configPath": ["xpack", "llmTasks"], + "requiredPlugins": ["inference", "productDocBase"], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/config.ts b/x-pack/plugins/ai_infra/llm_tasks/server/config.ts new file mode 100644 index 000000000000..c509af8bda64 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; + +const configSchema = schema.object({}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: {}, +}; + +export type LlmTasksConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/index.ts new file mode 100644 index 000000000000..1b18426dc2c3 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; +import type { LlmTasksConfig } from './config'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { LlmTasksPlugin } from './plugin'; + +export { config } from './config'; + +export type { LlmTasksPluginSetup, LlmTasksPluginStart }; + +export const plugin: PluginInitializer< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new LlmTasksPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts new file mode 100644 index 000000000000..d10c495ece15 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import type { LlmTasksConfig } from './config'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { retrieveDocumentation } from './tasks'; + +export class LlmTasksPlugin + implements + Plugin< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies + > +{ + private logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + setupDependencies: PluginSetupDependencies + ): LlmTasksPluginSetup { + return {}; + } + + start(core: CoreStart, startDependencies: PluginStartDependencies): LlmTasksPluginStart { + const { inference, productDocBase } = startDependencies; + return { + retrieveDocumentationAvailable: async () => { + const docBaseStatus = await startDependencies.productDocBase.management.getStatus(); + return docBaseStatus.status === 'installed'; + }, + retrieveDocumentation: (options) => { + return retrieveDocumentation({ + outputAPI: inference.getClient({ request: options.request }).output, + searchDocAPI: productDocBase.search, + logger: this.logger.get('tasks.retrieve-documentation'), + })(options); + }, + }; + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts new file mode 100644 index 000000000000..41d391182344 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { retrieveDocumentation } from './retrieve_documentation'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts new file mode 100644 index 000000000000..22bf0745bd77 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { retrieveDocumentation } from './retrieve_documentation'; +export type { + RetrieveDocumentationAPI, + RetrieveDocumentationResult, + RetrieveDocumentationParams, +} from './types'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts new file mode 100644 index 000000000000..5722b73ca039 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServerMock } from '@kbn/core/server/mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { DocSearchResult } from '@kbn/product-doc-base-plugin/server/services/search'; + +import { retrieveDocumentation } from './retrieve_documentation'; +import { truncate, count as countTokens } from '../../utils/tokens'; +jest.mock('../../utils/tokens'); +const truncateMock = truncate as jest.MockedFn; +const countTokensMock = countTokens as jest.MockedFn; + +import { summarizeDocument } from './summarize_document'; +jest.mock('./summarize_document'); +const summarizeDocumentMock = summarizeDocument as jest.MockedFn; + +describe('retrieveDocumentation', () => { + let logger: MockedLogger; + let request: ReturnType; + let outputAPI: jest.Mock; + let searchDocAPI: jest.Mock; + let retrieve: ReturnType; + + const createResult = (parts: Partial = {}): DocSearchResult => { + return { + title: 'title', + content: 'content', + url: 'url', + productName: 'kibana', + ...parts, + }; + }; + + beforeEach(() => { + logger = loggerMock.create(); + request = httpServerMock.createKibanaRequest(); + outputAPI = jest.fn(); + searchDocAPI = jest.fn(); + retrieve = retrieveDocumentation({ logger, searchDocAPI, outputAPI }); + }); + + afterEach(() => { + summarizeDocumentMock.mockReset(); + truncateMock.mockReset(); + countTokensMock.mockReset(); + }); + + it('calls the search API with the right parameters', async () => { + searchDocAPI.mockResolvedValue({ results: [] }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + products: ['kibana'], + request, + max: 5, + connectorId: '.my-connector', + functionCalling: 'simulated', + }); + + expect(result).toEqual({ + success: true, + documents: [], + }); + + expect(searchDocAPI).toHaveBeenCalledTimes(1); + expect(searchDocAPI).toHaveBeenCalledWith({ + query: 'What is Kibana?', + products: ['kibana'], + max: 5, + }); + }); + + it('reduces the document length using the truncate strategy', async () => { + searchDocAPI.mockResolvedValue({ + results: [ + createResult({ content: 'content-1' }), + createResult({ content: 'content-2' }), + createResult({ content: 'content-3' }), + ], + }); + + countTokensMock.mockImplementation((text) => { + if (text === 'content-2') { + return 150; + } else { + return 50; + } + }); + truncateMock.mockReturnValue('truncated'); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'truncate', + }); + + expect(result.documents.length).toEqual(3); + expect(result.documents[0].content).toEqual('content-1'); + expect(result.documents[1].content).toEqual('truncated'); + expect(result.documents[2].content).toEqual('content-3'); + + expect(truncateMock).toHaveBeenCalledTimes(1); + expect(truncateMock).toHaveBeenCalledWith('content-2', 100); + }); + + it('reduces the document length using the summarize strategy', async () => { + searchDocAPI.mockResolvedValue({ + results: [ + createResult({ content: 'content-1' }), + createResult({ content: 'content-2' }), + createResult({ content: 'content-3' }), + ], + }); + + countTokensMock.mockImplementation((text) => { + if (text === 'content-2') { + return 50; + } else { + return 150; + } + }); + truncateMock.mockImplementation((text) => text); + + summarizeDocumentMock.mockImplementation(({ documentContent }) => { + return Promise.resolve({ summary: `${documentContent}-summarized` }); + }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'summarize', + }); + + expect(result.documents.length).toEqual(3); + expect(result.documents[0].content).toEqual('content-1-summarized'); + expect(result.documents[1].content).toEqual('content-2'); + expect(result.documents[2].content).toEqual('content-3-summarized'); + + expect(truncateMock).toHaveBeenCalledTimes(2); + expect(truncateMock).toHaveBeenCalledWith('content-1-summarized', 100); + expect(truncateMock).toHaveBeenCalledWith('content-3-summarized', 100); + }); + + it('logs an error and return an empty list of docs in case of error', async () => { + searchDocAPI.mockResolvedValue({ + results: [createResult({ content: 'content-1' })], + }); + countTokensMock.mockImplementation(() => { + return 150; + }); + summarizeDocumentMock.mockImplementation(() => { + throw new Error('woups'); + }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'summarize', + }); + + expect(result).toEqual({ + success: false, + documents: [], + }); + + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Error retrieving documentation') + ); + }); +}); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts new file mode 100644 index 000000000000..96f966e48360 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { OutputAPI } from '@kbn/inference-common'; +import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; +import { truncate, count as countTokens } from '../../utils/tokens'; +import type { RetrieveDocumentationAPI } from './types'; +import { summarizeDocument } from './summarize_document'; + +const MAX_DOCUMENTS_DEFAULT = 3; +const MAX_TOKENS_DEFAULT = 1000; + +export const retrieveDocumentation = + ({ + outputAPI, + searchDocAPI, + logger: log, + }: { + outputAPI: OutputAPI; + searchDocAPI: ProductDocSearchAPI; + logger: Logger; + }): RetrieveDocumentationAPI => + async ({ + searchTerm, + connectorId, + products, + functionCalling, + max = MAX_DOCUMENTS_DEFAULT, + maxDocumentTokens = MAX_TOKENS_DEFAULT, + tokenReductionStrategy = 'summarize', + }) => { + try { + const { results } = await searchDocAPI({ query: searchTerm, products, max }); + + log.debug(`searching with term=[${searchTerm}] returned ${results.length} documents`); + + const processedDocuments = await Promise.all( + results.map(async (document) => { + const tokenCount = countTokens(document.content); + const docHasTooManyTokens = tokenCount >= maxDocumentTokens; + log.debug( + `processing doc [${document.url}] - tokens : [${tokenCount}] - tooManyTokens: [${docHasTooManyTokens}]` + ); + + let content = document.content; + if (docHasTooManyTokens) { + if (tokenReductionStrategy === 'summarize') { + const extractResponse = await summarizeDocument({ + searchTerm, + documentContent: document.content, + outputAPI, + connectorId, + functionCalling, + }); + content = truncate(extractResponse.summary, maxDocumentTokens); + } else { + content = truncate(document.content, maxDocumentTokens); + } + } + + log.debug(`done processing document [${document.url}]`); + return { + title: document.title, + url: document.url, + content, + }; + }) + ); + + log.debug(() => { + const docsAsJson = JSON.stringify(processedDocuments); + return `searching with term=[${searchTerm}] - results: ${docsAsJson}`; + }); + + return { + success: true, + documents: processedDocuments.filter((doc) => doc.content.length > 0), + }; + } catch (e) { + log.error(`Error retrieving documentation: ${e.message}. Returning empty results.`); + return { success: false, documents: [] }; + } + }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts new file mode 100644 index 000000000000..815cbc94d08f --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolSchema, FunctionCallingMode, OutputAPI } from '@kbn/inference-common'; + +const summarizeDocumentSchema = { + type: 'object', + properties: { + useful: { + type: 'boolean', + description: `Whether the provided document has any useful information related to the user's query.`, + }, + summary: { + type: 'string', + description: `The condensed version of the document that can be used to answer the question. Can be empty.`, + }, + }, + required: ['useful'], +} as const satisfies ToolSchema; + +interface SummarizeDocumentResponse { + summary: string; +} + +export const summarizeDocument = async ({ + searchTerm, + documentContent, + connectorId, + outputAPI, + functionCalling, +}: { + searchTerm: string; + documentContent: string; + outputAPI: OutputAPI; + connectorId: string; + functionCalling?: FunctionCallingMode; +}): Promise => { + const result = await outputAPI({ + id: 'summarize_document', + connectorId, + functionCalling, + system: `You are an helpful Elastic assistant, and your current task is to help answer the user's question. + + Given a question and a document, please provide a condensed version of the document that can be used to answer the question. + - Limit the length of the output to 500 words. + - Try to include all relevant information that could be used to answer the question. If this + can't be done within the 500 words limit, then only include the most relevant information related to the question. + - If you think the document isn't relevant at all to answer the question, just return an empty text`, + input: ` + ## User question + + ${searchTerm} + + ## Document + + ${documentContent} + `, + schema: summarizeDocumentSchema, + }); + return { + summary: result.output.summary ?? '', + }; +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts new file mode 100644 index 000000000000..1e0637fcd344 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRequest } from '@kbn/core/server'; +import type { FunctionCallingMode } from '@kbn/inference-common'; +import type { ProductName } from '@kbn/product-doc-common'; + +/** + * Parameters for {@link RetrieveDocumentationAPI} + */ +export interface RetrieveDocumentationParams { + /** + * The search term to perform semantic text with. + * E.g. "What is Kibana Lens?" + */ + searchTerm: string; + /** + * Maximum number of documents to return. + * Defaults to 3. + */ + max?: number; + /** + * Optional list of products to restrict the search to. + */ + products?: ProductName[]; + /** + * The maximum number of tokens to return *per document*. + * Documents exceeding this limit will go through token reduction. + * + * Defaults to `1000`. + */ + maxDocumentTokens?: number; + /** + * The token reduction strategy to apply for documents exceeding max token count. + * - truncate: Will keep the N first tokens + * - summarize: Will call the LLM asking to generate a contextualized summary of the document + * + * Overall, `summarize` is way more efficient, but significantly slower, given that an additional + * LLM call will be performed. + * + * Defaults to `summarize` + */ + tokenReductionStrategy?: 'truncate' | 'summarize'; + /** + * The request that initiated the task. + */ + request: KibanaRequest; + /** + * Id of the LLM connector to use for the task. + */ + connectorId: string; + functionCalling?: FunctionCallingMode; +} + +export interface RetrievedDocument { + title: string; + url: string; + content: string; +} + +export interface RetrieveDocumentationResult { + success: boolean; + documents: RetrievedDocument[]; +} + +export type RetrieveDocumentationAPI = ( + options: RetrieveDocumentationParams +) => Promise; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts new file mode 100644 index 000000000000..d550e4398b50 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; +import type { ProductDocBaseStartContract } from '@kbn/product-doc-base-plugin/server'; +import type { RetrieveDocumentationAPI } from './tasks/retrieve_documentation'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PluginSetupDependencies {} + +export interface PluginStartDependencies { + inference: InferenceServerStart; + productDocBase: ProductDocBaseStartContract; +} + +/** + * Describes public llmTasks plugin contract returned at the `setup` stage. + */ +export interface LlmTasksPluginSetup {} + +/** + * Describes public llmTasks plugin contract returned at the `start` stage. + */ +export interface LlmTasksPluginStart { + /** + * Checks if all prerequisites to use the `retrieveDocumentation` task + * are respected. Can be used to check if the task can be registered + * as LLM tool for example. + */ + retrieveDocumentationAvailable: () => Promise; + /** + * Perform the `retrieveDocumentation` task. + * + * @see RetrieveDocumentationAPI + */ + retrieveDocumentation: RetrieveDocumentationAPI; +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts new file mode 100644 index 000000000000..dce97eaea9b7 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { count, truncate } from './tokens'; + +describe('count', () => { + it('returns the token count of a given text', () => { + expect(count('some short sentence')).toBeGreaterThan(1); + }); +}); + +describe('truncate', () => { + it('truncates text that exceed the specified maximum token count', () => { + const text = 'some sentence that is likely longer than 5 tokens.'; + const output = truncate(text, 5); + expect(output.length).toBeLessThan(text.length); + }); + it('keeps text with a smaller amount of tokens unchanged', () => { + const text = 'some sentence that is likely less than 100 tokens.'; + const output = truncate(text, 100); + expect(output.length).toEqual(text.length); + }); +}); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts new file mode 100644 index 000000000000..cb469144255b --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { encode, decode } from 'gpt-tokenizer'; + +export const count = (text: string): number => { + return encode(text).length; +}; + +export const truncate = (text: string, maxTokens: number): string => { + const encoded = encode(text); + if (encoded.length > maxTokens) { + const truncated = encoded.slice(0, maxTokens); + return decode(truncated); + } + return text; +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json new file mode 100644 index 000000000000..03b87827d941 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json @@ -0,0 +1,27 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + "scripts/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/config-schema", + "@kbn/product-doc-common", + "@kbn/inference-plugin", + "@kbn/product-doc-base-plugin", + "@kbn/logging-mocks", + "@kbn/inference-common", + ] +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/README.md b/x-pack/plugins/ai_infra/product_doc_base/README.md new file mode 100644 index 000000000000..0ff6c34dd278 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/README.md @@ -0,0 +1,3 @@ +# Product documentation base plugin + +This plugin contains the product documentation base service. diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts new file mode 100644 index 000000000000..1622df5ed865 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const productDocInstallStatusSavedObjectTypeName = 'product-doc-install-status'; + +/** + * The id of the inference endpoint we're creating for our product doc indices. + * Could be replaced with the default elser 2 endpoint once the default endpoint feature is available. + */ +export const internalElserInferenceId = 'kibana-internal-elser2'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts new file mode 100644 index 000000000000..0237bd2c3b48 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; +import type { ProductInstallState, InstallationStatus } from '../install_status'; + +export const INSTALLATION_STATUS_API_PATH = '/internal/product_doc_base/status'; +export const INSTALL_ALL_API_PATH = '/internal/product_doc_base/install'; +export const UNINSTALL_ALL_API_PATH = '/internal/product_doc_base/uninstall'; + +export interface InstallationStatusResponse { + overall: InstallationStatus; + perProducts: Record; +} + +export interface PerformInstallResponse { + installed: boolean; +} + +export interface UninstallResponse { + success: boolean; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts new file mode 100644 index 000000000000..81102d43c1ff --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; + +export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; + +/** + * DTO representation of the product doc install status SO + */ +export interface ProductDocInstallStatus { + id: string; + productName: ProductName; + productVersion: string; + installationStatus: InstallationStatus; + lastInstallationDate: Date | undefined; + lastInstallationFailureReason: string | undefined; + indexName?: string; +} + +export interface ProductInstallState { + status: InstallationStatus; + version?: string; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/jest.config.js b/x-pack/plugins/ai_infra/product_doc_base/jest.config.js new file mode 100644 index 000000000000..fc06be251a6f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/jest.config.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: [ + '/x-pack/plugins/ai_infra/product_doc_base/public', + '/x-pack/plugins/ai_infra/product_doc_base/server', + '/x-pack/plugins/ai_infra/product_doc_base/common', + ], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/ai_infra/product_doc_base/{public,server,common}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc new file mode 100644 index 000000000000..268b4a70c992 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc @@ -0,0 +1,15 @@ +{ + "type": "plugin", + "id": "@kbn/product-doc-base-plugin", + "owner": "@elastic/appex-ai-infra", + "plugin": { + "id": "productDocBase", + "server": true, + "browser": true, + "configPath": ["xpack", "productDocBase"], + "requiredPlugins": ["licensing", "taskManager"], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts new file mode 100644 index 000000000000..b5ccbf029a73 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { ProductDocBasePlugin } from './plugin'; +import type { + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies, + PublicPluginConfig, +} from './types'; + +export type { ProductDocBasePluginSetup, ProductDocBasePluginStart }; + +export const plugin: PluginInitializer< + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies +> = (pluginInitializerContext: PluginInitializerContext) => + new ProductDocBasePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx new file mode 100644 index 000000000000..6f2c989b6e45 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; +import type { Logger } from '@kbn/logging'; +import type { + PublicPluginConfig, + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { InstallationService } from './services/installation'; + +export class ProductDocBasePlugin + implements + Plugin< + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + pluginsSetup: PluginSetupDependencies + ): ProductDocBasePluginSetup { + return {}; + } + + start(coreStart: CoreStart, pluginsStart: PluginStartDependencies): ProductDocBasePluginStart { + const installationService = new InstallationService({ http: coreStart.http }); + + return { + installation: { + getStatus: () => installationService.getInstallationStatus(), + install: () => installationService.install(), + uninstall: () => installationService.uninstall(), + }, + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts new file mode 100644 index 000000000000..2eee8613d77d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { InstallationService } from './installation_service'; +export type { InstallationAPI } from './types'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts new file mode 100644 index 000000000000..294aeb99e0fd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServiceMock } from '@kbn/core/public/mocks'; +import { InstallationService } from './installation_service'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, +} from '../../../common/http_api/installation'; + +describe('InstallationService', () => { + let http: ReturnType; + let service: InstallationService; + + beforeEach(() => { + http = httpServiceMock.createSetupContract(); + service = new InstallationService({ http }); + }); + + describe('#getInstallationStatus', () => { + it('calls the endpoint with the right parameters', async () => { + await service.getInstallationStatus(); + expect(http.get).toHaveBeenCalledTimes(1); + expect(http.get).toHaveBeenCalledWith(INSTALLATION_STATUS_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.get.mockResolvedValue(expected); + + const response = await service.getInstallationStatus(); + expect(response).toEqual(expected); + }); + }); + describe('#install', () => { + beforeEach(() => { + http.post.mockResolvedValue({ installed: true }); + }); + + it('calls the endpoint with the right parameters', async () => { + await service.install(); + expect(http.post).toHaveBeenCalledTimes(1); + expect(http.post).toHaveBeenCalledWith(INSTALL_ALL_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { installed: true }; + http.post.mockResolvedValue(expected); + + const response = await service.install(); + expect(response).toEqual(expected); + }); + it('throws when the server returns installed: false', async () => { + const expected = { installed: false }; + http.post.mockResolvedValue(expected); + + await expect(service.install()).rejects.toThrowErrorMatchingInlineSnapshot( + `"Installation did not complete successfully"` + ); + }); + }); + describe('#uninstall', () => { + it('calls the endpoint with the right parameters', async () => { + await service.uninstall(); + expect(http.post).toHaveBeenCalledTimes(1); + expect(http.post).toHaveBeenCalledWith(UNINSTALL_ALL_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.post.mockResolvedValue(expected); + + const response = await service.uninstall(); + expect(response).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts new file mode 100644 index 000000000000..ff347f52cb53 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HttpSetup } from '@kbn/core-http-browser'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../../common/http_api/installation'; + +export class InstallationService { + private readonly http: HttpSetup; + + constructor({ http }: { http: HttpSetup }) { + this.http = http; + } + + async getInstallationStatus(): Promise { + return await this.http.get(INSTALLATION_STATUS_API_PATH); + } + + async install(): Promise { + const response = await this.http.post(INSTALL_ALL_API_PATH); + if (!response.installed) { + throw new Error('Installation did not complete successfully'); + } + return response; + } + + async uninstall(): Promise { + return await this.http.post(UNINSTALL_ALL_API_PATH); + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts new file mode 100644 index 000000000000..5c01c84b2462 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../../common/http_api/installation'; + +export interface InstallationAPI { + getStatus(): Promise; + install(): Promise; + uninstall(): Promise; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts new file mode 100644 index 000000000000..1d06b0e08fa2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallationAPI } from './services/installation'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PublicPluginConfig {} + +export interface PluginSetupDependencies {} + +export interface PluginStartDependencies {} + +export interface ProductDocBasePluginSetup {} + +export interface ProductDocBasePluginStart { + installation: InstallationAPI; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/config.ts b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts new file mode 100644 index 000000000000..bd0892d58270 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; + +const configSchema = schema.object({ + artifactRepositoryUrl: schema.string({ + defaultValue: 'https://kibana-knowledge-base-artifacts.elastic.co', + }), +}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: {}, +}; + +export type ProductDocBaseConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts new file mode 100644 index 000000000000..805a0f2ea8c4 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; +import type { ProductDocBaseConfig } from './config'; +import type { + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies, +} from './types'; +import { ProductDocBasePlugin } from './plugin'; + +export { config } from './config'; + +export type { ProductDocBaseSetupContract, ProductDocBaseStartContract }; +export type { SearchApi as ProductDocSearchAPI } from './services/search/types'; + +export const plugin: PluginInitializer< + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new ProductDocBasePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts new file mode 100644 index 000000000000..bd5d6a720dd7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock } from '@kbn/core/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; +import { ProductDocBasePlugin } from './plugin'; +import { ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies } from './types'; + +jest.mock('./services/package_installer'); +jest.mock('./services/search'); +jest.mock('./services/doc_install_status'); +jest.mock('./routes'); +jest.mock('./tasks'); +import { registerRoutes } from './routes'; +import { PackageInstaller } from './services/package_installer'; +import { registerTaskDefinitions, scheduleEnsureUpToDateTask } from './tasks'; + +const PackageInstallMock = PackageInstaller as jest.Mock; + +describe('ProductDocBasePlugin', () => { + let initContext: ReturnType; + let plugin: ProductDocBasePlugin; + let pluginSetupDeps: ProductDocBaseSetupDependencies; + let pluginStartDeps: ProductDocBaseStartDependencies; + + beforeEach(() => { + initContext = coreMock.createPluginInitializerContext(); + plugin = new ProductDocBasePlugin(initContext); + pluginSetupDeps = { + taskManager: taskManagerMock.createSetup(), + }; + pluginStartDeps = { + licensing: licensingMock.createStart(), + taskManager: taskManagerMock.createStart(), + }; + + PackageInstallMock.mockReturnValue({ ensureUpToDate: jest.fn().mockResolvedValue({}) }); + }); + + afterEach(() => { + (scheduleEnsureUpToDateTask as jest.Mock).mockReset(); + }); + + describe('#setup', () => { + it('register the routes', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + + expect(registerRoutes).toHaveBeenCalledTimes(1); + }); + it('register the product-doc SO type', () => { + const coreSetup = coreMock.createSetup(); + plugin.setup(coreSetup, pluginSetupDeps); + + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledTimes(1); + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith( + expect.objectContaining({ + name: productDocInstallStatusSavedObjectTypeName, + }) + ); + }); + it('register the task definitions', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + + expect(registerTaskDefinitions).toHaveBeenCalledTimes(3); + }); + }); + + describe('#start', () => { + it('returns a contract with the expected shape', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + const startContract = plugin.start(coreMock.createStart(), pluginStartDeps); + expect(startContract).toEqual({ + management: { + getStatus: expect.any(Function), + install: expect.any(Function), + uninstall: expect.any(Function), + update: expect.any(Function), + }, + search: expect.any(Function), + }); + }); + + it('schedules the update task', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + plugin.start(coreMock.createStart(), pluginStartDeps); + + expect(scheduleEnsureUpToDateTask).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts new file mode 100644 index 000000000000..c8ed100cabb1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import type { Logger } from '@kbn/logging'; +import { getDataPath } from '@kbn/utils'; +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import { SavedObjectsClient } from '@kbn/core/server'; +import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; +import type { ProductDocBaseConfig } from './config'; +import { + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies, + InternalServices, +} from './types'; +import { productDocInstallStatusSavedObjectType } from './saved_objects'; +import { PackageInstaller } from './services/package_installer'; +import { InferenceEndpointManager } from './services/inference_endpoint'; +import { ProductDocInstallClient } from './services/doc_install_status'; +import { DocumentationManager } from './services/doc_manager'; +import { SearchService } from './services/search'; +import { registerRoutes } from './routes'; +import { registerTaskDefinitions } from './tasks'; + +export class ProductDocBasePlugin + implements + Plugin< + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies + > +{ + private logger: Logger; + private internalServices?: InternalServices; + + constructor(private readonly context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + { taskManager }: ProductDocBaseSetupDependencies + ): ProductDocBaseSetupContract { + const getServices = () => { + if (!this.internalServices) { + throw new Error('getServices called before #start'); + } + return this.internalServices; + }; + + coreSetup.savedObjects.registerType(productDocInstallStatusSavedObjectType); + + registerTaskDefinitions({ + taskManager, + getServices, + }); + + const router = coreSetup.http.createRouter(); + registerRoutes({ + router, + getServices, + }); + + return {}; + } + + start( + core: CoreStart, + { licensing, taskManager }: ProductDocBaseStartDependencies + ): ProductDocBaseStartContract { + const soClient = new SavedObjectsClient( + core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) + ); + const productDocClient = new ProductDocInstallClient({ soClient }); + + const endpointManager = new InferenceEndpointManager({ + esClient: core.elasticsearch.client.asInternalUser, + logger: this.logger.get('endpoint-manager'), + }); + + const packageInstaller = new PackageInstaller({ + esClient: core.elasticsearch.client.asInternalUser, + productDocClient, + endpointManager, + kibanaVersion: this.context.env.packageInfo.version, + artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'), + artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl, + logger: this.logger.get('package-installer'), + }); + + const searchService = new SearchService({ + esClient: core.elasticsearch.client.asInternalUser, + logger: this.logger.get('search-service'), + }); + + const documentationManager = new DocumentationManager({ + logger: this.logger.get('doc-manager'), + docInstallClient: productDocClient, + licensing, + taskManager, + auditService: core.security.audit, + }); + + this.internalServices = { + logger: this.logger, + packageInstaller, + installClient: productDocClient, + documentationManager, + licensing, + taskManager, + }; + + documentationManager.update().catch((err) => { + this.logger.error(`Error scheduling product documentation update task: ${err.message}`); + }); + + return { + management: { + install: documentationManager.install.bind(documentationManager), + update: documentationManager.update.bind(documentationManager), + uninstall: documentationManager.uninstall.bind(documentationManager), + getStatus: documentationManager.getStatus.bind(documentationManager), + }, + search: searchService.search.bind(searchService), + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts new file mode 100644 index 000000000000..66660c199d81 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import { registerInstallationRoutes } from './installation'; +import type { InternalServices } from '../types'; + +export const registerRoutes = ({ + router, + getServices, +}: { + router: IRouter; + getServices: () => InternalServices; +}) => { + registerInstallationRoutes({ getServices, router }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts new file mode 100644 index 000000000000..dbede9f7d94d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../common/http_api/installation'; +import type { InternalServices } from '../types'; + +export const registerInstallationRoutes = ({ + router, + getServices, +}: { + router: IRouter; + getServices: () => InternalServices; +}) => { + router.get( + { + path: INSTALLATION_STATUS_API_PATH, + validate: false, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + }, + }, + async (ctx, req, res) => { + const { installClient, documentationManager } = getServices(); + const installStatus = await installClient.getInstallationStatus(); + const { status: overallStatus } = await documentationManager.getStatus(); + + return res.ok({ + body: { + perProducts: installStatus, + overall: overallStatus, + }, + }); + } + ); + + router.post( + { + path: INSTALL_ALL_API_PATH, + validate: false, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + timeout: { idleSocket: 20 * 60 * 1000 }, // install can take time. + }, + }, + async (ctx, req, res) => { + const { documentationManager } = getServices(); + + await documentationManager.install({ + request: req, + force: false, + wait: true, + }); + + // check status after installation in case of failure + const { status } = await documentationManager.getStatus(); + + return res.ok({ + body: { + installed: status === 'installed', + }, + }); + } + ); + + router.post( + { + path: UNINSTALL_ALL_API_PATH, + validate: false, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + }, + }, + async (ctx, req, res) => { + const { documentationManager } = getServices(); + + await documentationManager.uninstall({ + request: req, + wait: true, + }); + + return res.ok({ + body: { + success: true, + }, + }); + } + ); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts new file mode 100644 index 000000000000..f87c6d37eb66 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { + productDocInstallStatusSavedObjectType, + type ProductDocInstallStatusAttributes, +} from './product_doc_install'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts new file mode 100644 index 000000000000..47cf7eb50cdd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsType } from '@kbn/core/server'; +import type { ProductName } from '@kbn/product-doc-common'; +import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts'; +import type { InstallationStatus } from '../../common/install_status'; + +/** + * Interface describing the raw attributes of the product doc install SO type. + * Contains more fields than the mappings, which only list + * indexed fields. + */ +export interface ProductDocInstallStatusAttributes { + product_name: ProductName; + product_version: string; + installation_status: InstallationStatus; + last_installation_date?: number; + last_installation_failure_reason?: string; + index_name?: string; +} + +export const productDocInstallStatusSavedObjectType: SavedObjectsType = + { + name: productDocInstallStatusSavedObjectTypeName, + hidden: true, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + product_name: { type: 'keyword' }, + product_version: { type: 'keyword' }, + installation_status: { type: 'keyword' }, + last_installation_date: { type: 'date' }, + index_name: { type: 'keyword' }, + }, + }, + management: { + importableAndExportable: false, + }, + modelVersions: {}, + }; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts new file mode 100644 index 000000000000..d55cb303b190 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ProductDocInstallClient } from './product_doc_install_service'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts new file mode 100644 index 000000000000..6460d8452dc2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; +import { soToModel } from './model_conversion'; + +const createObj = ( + attrs: ProductDocInstallStatusAttributes +): SavedObject => { + return { + id: 'some-id', + type: 'product-doc-install-status', + attributes: attrs, + references: [], + }; +}; + +describe('soToModel', () => { + it('converts the SO to the expected shape', () => { + const input = createObj({ + product_name: 'kibana', + product_version: '8.16', + installation_status: 'installed', + last_installation_date: 9000, + index_name: '.kibana', + }); + + const output = soToModel(input); + + expect(output).toEqual({ + id: 'some-id', + productName: 'kibana', + productVersion: '8.16', + indexName: '.kibana', + installationStatus: 'installed', + lastInstallationDate: expect.any(Date), + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts new file mode 100644 index 000000000000..cf77bb9222a1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { ProductDocInstallStatus } from '../../../common/install_status'; +import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; + +export const soToModel = ( + so: SavedObject +): ProductDocInstallStatus => { + return { + id: so.id, + productName: so.attributes.product_name, + productVersion: so.attributes.product_version, + installationStatus: so.attributes.installation_status, + indexName: so.attributes.index_name, + lastInstallationDate: so.attributes.last_installation_date + ? new Date(so.attributes.last_installation_date) + : undefined, + lastInstallationFailureReason: so.attributes.last_installation_failure_reason, + }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts new file mode 100644 index 000000000000..81249038a129 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from '@kbn/core/server'; +import { DocumentationProduct } from '@kbn/product-doc-common'; +import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { ProductDocInstallClient } from './product_doc_install_service'; + +const createObj = (attrs: TypeAttributes): SavedObjectsFindResult => { + return { + id: attrs.product_name, + type: 'type', + references: [], + attributes: attrs, + score: 42, + }; +}; + +describe('ProductDocInstallClient', () => { + let soClient: ReturnType; + let service: ProductDocInstallClient; + + beforeEach(() => { + soClient = savedObjectsClientMock.create(); + service = new ProductDocInstallClient({ soClient }); + }); + + describe('getInstallationStatus', () => { + it('returns the installation status based on existing entries', async () => { + soClient.find.mockResolvedValue({ + saved_objects: [ + createObj({ + product_name: 'kibana', + product_version: '8.15', + installation_status: 'installed', + }), + createObj({ + product_name: 'elasticsearch', + product_version: '8.15', + installation_status: 'installing', + }), + ], + total: 2, + per_page: 100, + page: 1, + }); + + const installStatus = await service.getInstallationStatus(); + + expect(Object.keys(installStatus).sort()).toEqual(Object.keys(DocumentationProduct).sort()); + expect(installStatus.kibana).toEqual({ + status: 'installed', + version: '8.15', + }); + expect(installStatus.security).toEqual({ + status: 'uninstalled', + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts new file mode 100644 index 000000000000..24625ebc5158 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; +import { ProductName, DocumentationProduct } from '@kbn/product-doc-common'; +import type { ProductInstallState } from '../../../common/install_status'; +import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; +import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; + +export class ProductDocInstallClient { + private soClient: SavedObjectsClientContract; + + constructor({ soClient }: { soClient: SavedObjectsClientContract }) { + this.soClient = soClient; + } + + async getInstallationStatus(): Promise> { + const response = await this.soClient.find({ + type: typeName, + perPage: 100, + }); + + const installStatus = Object.values(DocumentationProduct).reduce((memo, product) => { + memo[product] = { status: 'uninstalled' }; + return memo; + }, {} as Record); + + response.saved_objects.forEach(({ attributes }) => { + installStatus[attributes.product_name as ProductName] = { + status: attributes.installation_status, + version: attributes.product_version, + }; + }); + + return installStatus; + } + + async setInstallationStarted(fields: { productName: ProductName; productVersion: string }) { + const { productName, productVersion } = fields; + const objectId = getObjectIdFromProductName(productName); + const attributes = { + product_name: productName, + product_version: productVersion, + installation_status: 'installing' as const, + last_installation_failure_reason: '', + }; + await this.soClient.update(typeName, objectId, attributes, { + upsert: attributes, + }); + } + + async setInstallationSuccessful(productName: ProductName, indexName: string) { + const objectId = getObjectIdFromProductName(productName); + await this.soClient.update(typeName, objectId, { + installation_status: 'installed', + index_name: indexName, + }); + } + + async setInstallationFailed(productName: ProductName, failureReason: string) { + const objectId = getObjectIdFromProductName(productName); + await this.soClient.update(typeName, objectId, { + installation_status: 'error', + last_installation_failure_reason: failureReason, + }); + } + + async setUninstalled(productName: ProductName) { + const objectId = getObjectIdFromProductName(productName); + try { + await this.soClient.update(typeName, objectId, { + installation_status: 'uninstalled', + last_installation_failure_reason: '', + }); + } catch (e) { + if (!SavedObjectsErrorHelpers.isNotFoundError(e)) { + throw e; + } + } + } +} + +const getObjectIdFromProductName = (productName: ProductName) => + `kb-product-doc-${productName}-status`.toLowerCase(); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts new file mode 100644 index 000000000000..c2a0adbac9f2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductDocInstallClient } from './product_doc_install_service'; + +export type InstallClientMock = jest.Mocked; + +const createInstallClientMock = (): InstallClientMock => { + return { + getInstallationStatus: jest.fn(), + setInstallationStarted: jest.fn(), + setInstallationSuccessful: jest.fn(), + setInstallationFailed: jest.fn(), + setUninstalled: jest.fn(), + } as unknown as InstallClientMock; +}; + +export const installClientMock = { + create: createInstallClientMock, +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts new file mode 100644 index 000000000000..d4af5b7ebdb2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ILicense } from '@kbn/licensing-plugin/server'; + +export const checkLicense = (license: ILicense): boolean => { + const result = license.check('elastic documentation', 'enterprise'); + return result.state === 'valid'; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts new file mode 100644 index 000000000000..0be913ee6dd7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { securityServiceMock, httpServerMock } from '@kbn/core/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import { DocumentationManager } from './doc_manager'; + +jest.mock('../../tasks'); +import { + scheduleInstallAllTask, + scheduleUninstallAllTask, + scheduleEnsureUpToDateTask, + getTaskStatus, + waitUntilTaskCompleted, +} from '../../tasks'; + +const scheduleInstallAllTaskMock = scheduleInstallAllTask as jest.MockedFn< + typeof scheduleInstallAllTask +>; +const scheduleUninstallAllTaskMock = scheduleUninstallAllTask as jest.MockedFn< + typeof scheduleUninstallAllTask +>; +const scheduleEnsureUpToDateTaskMock = scheduleEnsureUpToDateTask as jest.MockedFn< + typeof scheduleEnsureUpToDateTask +>; +const waitUntilTaskCompletedMock = waitUntilTaskCompleted as jest.MockedFn< + typeof waitUntilTaskCompleted +>; +const getTaskStatusMock = getTaskStatus as jest.MockedFn; + +describe('DocumentationManager', () => { + let logger: MockedLogger; + let taskManager: ReturnType; + let licensing: ReturnType; + let auditService: ReturnType['audit']; + let docInstallClient: jest.Mocked; + + let docManager: DocumentationManager; + + beforeEach(() => { + logger = loggerMock.create(); + taskManager = taskManagerMock.createStart(); + licensing = licensingMock.createStart(); + auditService = securityServiceMock.createStart().audit; + + docInstallClient = { + getInstallationStatus: jest.fn(), + } as unknown as jest.Mocked; + + docManager = new DocumentationManager({ + logger, + taskManager, + licensing, + auditService, + docInstallClient, + }); + }); + + afterEach(() => { + scheduleInstallAllTaskMock.mockReset(); + scheduleUninstallAllTaskMock.mockReset(); + scheduleEnsureUpToDateTaskMock.mockReset(); + waitUntilTaskCompletedMock.mockReset(); + getTaskStatusMock.mockReset(); + }); + + describe('#install', () => { + beforeEach(() => { + licensing.getLicense.mockResolvedValue( + licensingMock.createLicense({ license: { type: 'enterprise' } }) + ); + + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleInstallAllTask`', async () => { + await docManager.install({}); + + expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleInstallAllTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.install({ wait: true }); + + expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('does not call scheduleInstallAllTask if already installed and not force', async () => { + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'installed' }, + } as Awaited>); + + await docManager.install({ wait: true }); + + expect(scheduleInstallAllTaskMock).not.toHaveBeenCalled(); + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.install({ force: false, wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_create', + category: ['database'], + type: ['creation'], + outcome: 'unknown', + }, + }); + }); + + it('throws an error if license level is not sufficient', async () => { + licensing.getLicense.mockResolvedValue( + licensingMock.createLicense({ license: { type: 'basic' } }) + ); + + await expect( + docManager.install({ force: false, wait: false }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Elastic documentation requires an enterprise license"` + ); + }); + }); + + describe('#update', () => { + beforeEach(() => { + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleEnsureUpToDateTask`', async () => { + await docManager.update({}); + + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.update({ wait: true }); + + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.update({ wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_update', + category: ['database'], + type: ['change'], + outcome: 'unknown', + }, + }); + }); + }); + + describe('#uninstall', () => { + beforeEach(() => { + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleUninstallAllTask`', async () => { + await docManager.uninstall({}); + + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.uninstall({ wait: true }); + + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.uninstall({ wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_delete', + category: ['database'], + type: ['deletion'], + outcome: 'unknown', + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts new file mode 100644 index 000000000000..40dc53e19cee --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { CoreAuditService } from '@kbn/core/server'; +import { type TaskManagerStartContract, TaskStatus } from '@kbn/task-manager-plugin/server'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { InstallationStatus } from '../../../common/install_status'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import { + INSTALL_ALL_TASK_ID, + scheduleInstallAllTask, + scheduleUninstallAllTask, + scheduleEnsureUpToDateTask, + getTaskStatus, + waitUntilTaskCompleted, +} from '../../tasks'; +import { checkLicense } from './check_license'; +import type { + DocumentationManagerAPI, + DocGetStatusResponse, + DocInstallOptions, + DocUninstallOptions, + DocUpdateOptions, +} from './types'; + +const TEN_MIN_IN_MS = 10 * 60 * 1000; + +/** + * High-level installation service, handling product documentation + * installation as unary operations, abstracting away the fact + * that documentation is composed of multiple entities. + */ +export class DocumentationManager implements DocumentationManagerAPI { + private logger: Logger; + private taskManager: TaskManagerStartContract; + private licensing: LicensingPluginStart; + private docInstallClient: ProductDocInstallClient; + private auditService: CoreAuditService; + + constructor({ + logger, + taskManager, + licensing, + docInstallClient, + auditService, + }: { + logger: Logger; + taskManager: TaskManagerStartContract; + licensing: LicensingPluginStart; + docInstallClient: ProductDocInstallClient; + auditService: CoreAuditService; + }) { + this.logger = logger; + this.taskManager = taskManager; + this.licensing = licensing; + this.docInstallClient = docInstallClient; + this.auditService = auditService; + } + + async install(options: DocInstallOptions = {}): Promise { + const { request, force = false, wait = false } = options; + + const { status } = await this.getStatus(); + if (!force && status === 'installed') { + return; + } + + const license = await this.licensing.getLicense(); + if (!checkLicense(license)) { + throw new Error('Elastic documentation requires an enterprise license'); + } + + const taskId = await scheduleInstallAllTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting installation of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_create', + category: ['database'], + type: ['creation'], + outcome: 'unknown', + }, + }); + } + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async update(options: DocUpdateOptions = {}): Promise { + const { request, wait = false } = options; + + const taskId = await scheduleEnsureUpToDateTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting update of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_update', + category: ['database'], + type: ['change'], + outcome: 'unknown', + }, + }); + } + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async uninstall(options: DocUninstallOptions = {}): Promise { + const { request, wait = false } = options; + + const taskId = await scheduleUninstallAllTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting deletion of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_delete', + category: ['database'], + type: ['deletion'], + outcome: 'unknown', + }, + }); + } + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async getStatus(): Promise { + const taskStatus = await getTaskStatus({ + taskManager: this.taskManager, + taskId: INSTALL_ALL_TASK_ID, + }); + if (taskStatus !== 'not_scheduled') { + const status = convertTaskStatus(taskStatus); + if (status !== 'unknown') { + return { status }; + } + } + + const installStatus = await this.docInstallClient.getInstallationStatus(); + const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); + return { status: overallStatus }; + } +} + +const convertTaskStatus = (taskStatus: TaskStatus): InstallationStatus | 'unknown' => { + switch (taskStatus) { + case TaskStatus.Idle: + case TaskStatus.Claiming: + case TaskStatus.Running: + return 'installing'; + case TaskStatus.Failed: + return 'error'; + case TaskStatus.Unrecognized: + case TaskStatus.DeadLetter: + case TaskStatus.ShouldDelete: + default: + return 'unknown'; + } +}; + +const getOverallStatus = (statuses: InstallationStatus[]): InstallationStatus => { + const statusOrder: InstallationStatus[] = ['error', 'installing', 'uninstalled', 'installed']; + for (const status of statusOrder) { + if (statuses.includes(status)) { + return status; + } + } + return 'installed'; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts new file mode 100644 index 000000000000..588b5e2f5cc6 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { DocumentationManager } from './doc_manager'; +export type { + DocumentationManagerAPI, + DocUninstallOptions, + DocInstallOptions, + DocUpdateOptions, + DocGetStatusResponse, +} from './types'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts new file mode 100644 index 000000000000..5a954a5ffb0f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRequest } from '@kbn/core/server'; +import type { InstallationStatus } from '../../../common/install_status'; + +/** + * APIs to manage the product documentation. + */ +export interface DocumentationManagerAPI { + /** + * Install the product documentation. + * By default, will only try to install if not already present. + * Can use the `force` option to forcefully reinstall. + */ + install(options?: DocInstallOptions): Promise; + /** + * Update the product documentation to the latest version. + * No-op if the product documentation is not currently installed. + */ + update(options?: DocUpdateOptions): Promise; + /** + * Uninstall the product documentation. + * No-op if the product documentation is not currently installed. + */ + uninstall(options?: DocUninstallOptions): Promise; + /** + * Returns the overall installation status of the documentation. + */ + getStatus(): Promise; +} + +/** + * Return type for {@link DocumentationManagerAPI.getStatus} + */ +export interface DocGetStatusResponse { + status: InstallationStatus; +} + +/** + * Options for {@link DocumentationManagerAPI.install} + */ +export interface DocInstallOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; + /** + * If true, will reinstall the documentation even if already present. + * Defaults to `false` + */ + force?: boolean; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} + +/** + * Options for {@link DocumentationManagerAPI.uninstall} + */ +export interface DocUninstallOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} + +/** + * Options for {@link DocumentationManagerAPI.update} + */ +export interface DocUpdateOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts new file mode 100644 index 000000000000..e5dabaaa9b7f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { InferenceEndpointManager } from './endpoint_manager'; + +jest.mock('./utils'); +import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils'; +const installElserMock = installElser as jest.MockedFn; +const getModelInstallStatusMock = getModelInstallStatus as jest.MockedFn< + typeof getModelInstallStatus +>; +const waitUntilModelDeployedMock = waitUntilModelDeployed as jest.MockedFn< + typeof waitUntilModelDeployed +>; + +describe('InferenceEndpointManager', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let endpointManager: InferenceEndpointManager; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + + endpointManager = new InferenceEndpointManager({ esClient, logger }); + }); + + afterEach(() => { + installElserMock.mockReset(); + getModelInstallStatusMock.mockReset(); + waitUntilModelDeployedMock.mockReset(); + }); + + describe('#ensureInternalElserInstalled', () => { + it('installs ELSER if not already installed', async () => { + getModelInstallStatusMock.mockResolvedValue({ installed: true }); + + await endpointManager.ensureInternalElserInstalled(); + + expect(installElserMock).not.toHaveBeenCalled(); + expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1); + }); + it('does not install ELSER if already present', async () => { + getModelInstallStatusMock.mockResolvedValue({ installed: false }); + + await endpointManager.ensureInternalElserInstalled(); + + expect(installElserMock).toHaveBeenCalledTimes(1); + expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts new file mode 100644 index 000000000000..4f7467501d61 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { internalElserInferenceId } from '../../../common/consts'; +import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils'; + +export class InferenceEndpointManager { + private readonly log: Logger; + private readonly esClient: ElasticsearchClient; + + constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) { + this.log = logger; + this.esClient = esClient; + } + + async ensureInternalElserInstalled() { + const { installed } = await getModelInstallStatus({ + inferenceId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + if (!installed) { + await installElser({ + inferenceId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + } + + await waitUntilModelDeployed({ + modelId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts new file mode 100644 index 000000000000..e4098ff58fe5 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { InferenceEndpointManager } from './endpoint_manager'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts new file mode 100644 index 000000000000..e9715c4ad2ac --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceEndpointManager } from './endpoint_manager'; + +export type InferenceEndpointManagerMock = jest.Mocked; + +const createMock = (): InferenceEndpointManagerMock => { + return { + ensureInternalElserInstalled: jest.fn(), + } as unknown as InferenceEndpointManagerMock; +}; + +export const inferenceManagerMock = { + create: createMock, +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts new file mode 100644 index 000000000000..be6caa34d0ad --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceTaskType } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const getModelInstallStatus = async ({ + inferenceId, + taskType = 'sparse_embedding', + client, +}: { + inferenceId: string; + taskType?: InferenceTaskType; + client: ElasticsearchClient; + log: Logger; +}) => { + const getInferenceRes = await client.inference.get( + { + task_type: taskType, + inference_id: inferenceId, + }, + { ignore: [404] } + ); + + const installed = (getInferenceRes.endpoints ?? []).some( + (endpoint) => endpoint.inference_id === inferenceId + ); + + return { installed }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts new file mode 100644 index 000000000000..089997557f30 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { waitUntilModelDeployed } from './wait_until_model_deployed'; +export { getModelInstallStatus } from './get_model_install_status'; +export { installElser } from './install_elser'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts new file mode 100644 index 000000000000..0e92d765a3d1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const installElser = async ({ + inferenceId, + client, + log, +}: { + inferenceId: string; + client: ElasticsearchClient; + log: Logger; +}) => { + await client.inference.put( + { + task_type: 'sparse_embedding', + inference_id: inferenceId, + inference_config: { + service: 'elasticsearch', + service_settings: { + num_allocations: 1, + num_threads: 1, + model_id: '.elser_model_2', + }, + task_settings: {}, + }, + }, + { requestTimeout: 5 * 60 * 1000 } + ); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts new file mode 100644 index 000000000000..83775ed80f5a --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const waitUntilModelDeployed = async ({ + modelId, + client, + log, + maxRetries = 20, + delay = 2000, +}: { + modelId: string; + client: ElasticsearchClient; + log: Logger; + maxRetries?: number; + delay?: number; +}) => { + for (let i = 0; i < maxRetries; i++) { + const statsRes = await client.ml.getTrainedModelsStats({ + model_id: modelId, + }); + const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats; + if (!deploymentStats || deploymentStats.nodes.length === 0) { + log.debug(`ML model [${modelId}] was not deployed - attempt ${i + 1} of ${maxRetries}`); + await sleep(delay); + continue; + } + return; + } + + throw new Error(`Timeout waiting for ML model ${modelId} to be deployed`); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts new file mode 100644 index 000000000000..a9edb7c38fda --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { PackageInstaller } from './package_installer'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts new file mode 100644 index 000000000000..3b7b7c234800 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const validateArtifactArchiveMock = jest.fn(); +export const fetchArtifactVersionsMock = jest.fn(); +export const createIndexMock = jest.fn(); +export const populateIndexMock = jest.fn(); + +jest.doMock('./steps', () => { + const actual = jest.requireActual('./steps'); + return { + ...actual, + validateArtifactArchive: validateArtifactArchiveMock, + fetchArtifactVersions: fetchArtifactVersionsMock, + createIndex: createIndexMock, + populateIndex: populateIndexMock, + }; +}); + +export const downloadToDiskMock = jest.fn(); +export const openZipArchiveMock = jest.fn(); +export const loadMappingFileMock = jest.fn(); + +jest.doMock('./utils', () => { + const actual = jest.requireActual('./utils'); + return { + ...actual, + downloadToDisk: downloadToDiskMock, + openZipArchive: openZipArchiveMock, + loadMappingFile: loadMappingFileMock, + }; +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts new file mode 100644 index 000000000000..e68bd0e9c505 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + downloadToDiskMock, + createIndexMock, + populateIndexMock, + loadMappingFileMock, + openZipArchiveMock, + validateArtifactArchiveMock, + fetchArtifactVersionsMock, +} from './package_installer.test.mocks'; + +import { + getArtifactName, + getProductDocIndexName, + DocumentationProduct, + ProductName, +} from '@kbn/product-doc-common'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { installClientMock } from '../doc_install_status/service.mock'; +import { inferenceManagerMock } from '../inference_endpoint/service.mock'; +import type { ProductInstallState } from '../../../common/install_status'; +import { PackageInstaller } from './package_installer'; + +const artifactsFolder = '/lost'; +const artifactRepositoryUrl = 'https://repository.com'; +const kibanaVersion = '8.16.3'; + +const callOrder = (fn: { mock: { invocationCallOrder: number[] } }): number => { + return fn.mock.invocationCallOrder[0]; +}; + +describe('PackageInstaller', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let productDocClient: ReturnType; + let endpointManager: ReturnType; + + let packageInstaller: PackageInstaller; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + productDocClient = installClientMock.create(); + endpointManager = inferenceManagerMock.create(); + packageInstaller = new PackageInstaller({ + artifactsFolder, + logger, + esClient, + productDocClient, + endpointManager, + artifactRepositoryUrl, + kibanaVersion, + }); + }); + + afterEach(() => { + downloadToDiskMock.mockReset(); + createIndexMock.mockReset(); + populateIndexMock.mockReset(); + loadMappingFileMock.mockReset(); + openZipArchiveMock.mockReset(); + validateArtifactArchiveMock.mockReset(); + fetchArtifactVersionsMock.mockReset(); + }); + + describe('installPackage', () => { + it('calls the steps with the right parameters', async () => { + const zipArchive = { + close: jest.fn(), + }; + openZipArchiveMock.mockResolvedValue(zipArchive); + + const mappings = Symbol('mappings'); + loadMappingFileMock.mockResolvedValue(mappings); + + await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }); + + const artifactName = getArtifactName({ + productName: 'kibana', + productVersion: '8.16', + }); + const indexName = getProductDocIndexName('kibana'); + expect(endpointManager.ensureInternalElserInstalled).toHaveBeenCalledTimes(1); + + expect(downloadToDiskMock).toHaveBeenCalledTimes(1); + expect(downloadToDiskMock).toHaveBeenCalledWith( + `${artifactRepositoryUrl}/${artifactName}`, + `${artifactsFolder}/${artifactName}` + ); + + expect(openZipArchiveMock).toHaveBeenCalledTimes(1); + expect(openZipArchiveMock).toHaveBeenCalledWith(`${artifactsFolder}/${artifactName}`); + + expect(loadMappingFileMock).toHaveBeenCalledTimes(1); + expect(loadMappingFileMock).toHaveBeenCalledWith(zipArchive); + + expect(createIndexMock).toHaveBeenCalledTimes(1); + expect(createIndexMock).toHaveBeenCalledWith({ + indexName, + mappings, + esClient, + log: logger, + }); + + expect(populateIndexMock).toHaveBeenCalledTimes(1); + expect(populateIndexMock).toHaveBeenCalledWith({ + indexName, + archive: zipArchive, + esClient, + log: logger, + }); + + expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledTimes(1); + expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledWith('kibana', indexName); + + expect(zipArchive.close).toHaveBeenCalledTimes(1); + + expect(productDocClient.setInstallationFailed).not.toHaveBeenCalled(); + }); + + it('executes the steps in the right order', async () => { + await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }); + + expect(callOrder(endpointManager.ensureInternalElserInstalled)).toBeLessThan( + callOrder(downloadToDiskMock) + ); + expect(callOrder(downloadToDiskMock)).toBeLessThan(callOrder(openZipArchiveMock)); + expect(callOrder(openZipArchiveMock)).toBeLessThan(callOrder(loadMappingFileMock)); + expect(callOrder(loadMappingFileMock)).toBeLessThan(callOrder(createIndexMock)); + expect(callOrder(createIndexMock)).toBeLessThan(callOrder(populateIndexMock)); + expect(callOrder(populateIndexMock)).toBeLessThan( + callOrder(productDocClient.setInstallationSuccessful) + ); + }); + + it('closes the archive and calls setInstallationFailed if the installation fails', async () => { + const zipArchive = { + close: jest.fn(), + }; + openZipArchiveMock.mockResolvedValue(zipArchive); + + populateIndexMock.mockImplementation(async () => { + throw new Error('something bad'); + }); + + await expect( + packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }) + ).rejects.toThrowError(); + + expect(productDocClient.setInstallationSuccessful).not.toHaveBeenCalled(); + + expect(zipArchive.close).toHaveBeenCalledTimes(1); + + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Error during documentation installation') + ); + + expect(productDocClient.setInstallationFailed).toHaveBeenCalledTimes(1); + expect(productDocClient.setInstallationFailed).toHaveBeenCalledWith( + 'kibana', + 'something bad' + ); + }); + }); + + describe('installALl', () => { + it('installs all the packages to their latest version', async () => { + jest.spyOn(packageInstaller, 'installPackage'); + + fetchArtifactVersionsMock.mockResolvedValue({ + kibana: ['8.15', '8.16'], + elasticsearch: ['8.15'], + }); + + await packageInstaller.installAll({}); + + expect(packageInstaller.installPackage).toHaveBeenCalledTimes(2); + + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'kibana', + productVersion: '8.16', + }); + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'elasticsearch', + productVersion: '8.15', + }); + }); + }); + + describe('ensureUpToDate', () => { + it('updates the installed packages to the latest version', async () => { + fetchArtifactVersionsMock.mockResolvedValue({ + kibana: ['8.15', '8.16'], + security: ['8.15', '8.16'], + elasticsearch: ['8.15'], + }); + + productDocClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'installed', version: '8.15' }, + security: { status: 'installed', version: '8.16' }, + elasticsearch: { status: 'uninstalled' }, + } as Record); + + jest.spyOn(packageInstaller, 'installPackage'); + + await packageInstaller.ensureUpToDate({}); + + expect(packageInstaller.installPackage).toHaveBeenCalledTimes(1); + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'kibana', + productVersion: '8.16', + }); + }); + }); + + describe('uninstallPackage', () => { + it('performs the uninstall steps', async () => { + await packageInstaller.uninstallPackage({ productName: 'kibana' }); + + expect(esClient.indices.delete).toHaveBeenCalledTimes(1); + expect(esClient.indices.delete).toHaveBeenCalledWith( + { + index: getProductDocIndexName('kibana'), + }, + expect.objectContaining({ ignore: [404] }) + ); + + expect(productDocClient.setUninstalled).toHaveBeenCalledTimes(1); + expect(productDocClient.setUninstalled).toHaveBeenCalledWith('kibana'); + }); + }); + + describe('uninstallAll', () => { + it('calls uninstall for all packages', async () => { + jest.spyOn(packageInstaller, 'uninstallPackage'); + + await packageInstaller.uninstallAll(); + + expect(packageInstaller.uninstallPackage).toHaveBeenCalledTimes( + Object.keys(DocumentationProduct).length + ); + Object.values(DocumentationProduct).forEach((productName) => { + expect(packageInstaller.uninstallPackage).toHaveBeenCalledWith({ productName }); + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts new file mode 100644 index 000000000000..7739219c15dc --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { + getArtifactName, + getProductDocIndexName, + DocumentationProduct, + type ProductName, +} from '@kbn/product-doc-common'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import type { InferenceEndpointManager } from '../inference_endpoint'; +import { downloadToDisk, openZipArchive, loadMappingFile, type ZipArchive } from './utils'; +import { majorMinor, latestVersion } from './utils/semver'; +import { + validateArtifactArchive, + fetchArtifactVersions, + createIndex, + populateIndex, +} from './steps'; + +interface PackageInstallerOpts { + artifactsFolder: string; + logger: Logger; + esClient: ElasticsearchClient; + productDocClient: ProductDocInstallClient; + endpointManager: InferenceEndpointManager; + artifactRepositoryUrl: string; + kibanaVersion: string; +} + +export class PackageInstaller { + private readonly log: Logger; + private readonly artifactsFolder: string; + private readonly esClient: ElasticsearchClient; + private readonly productDocClient: ProductDocInstallClient; + private readonly endpointManager: InferenceEndpointManager; + private readonly artifactRepositoryUrl: string; + private readonly currentVersion: string; + + constructor({ + artifactsFolder, + logger, + esClient, + productDocClient, + endpointManager, + artifactRepositoryUrl, + kibanaVersion, + }: PackageInstallerOpts) { + this.esClient = esClient; + this.productDocClient = productDocClient; + this.artifactsFolder = artifactsFolder; + this.endpointManager = endpointManager; + this.artifactRepositoryUrl = artifactRepositoryUrl; + this.currentVersion = majorMinor(kibanaVersion); + this.log = logger; + } + + /** + * Make sure that the currently installed doc packages are up to date. + * Will not upgrade products that are not already installed + */ + async ensureUpToDate({}: {}) { + const [repositoryVersions, installStatuses] = await Promise.all([ + fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }), + this.productDocClient.getInstallationStatus(), + ]); + + const toUpdate: Array<{ + productName: ProductName; + productVersion: string; + }> = []; + Object.entries(installStatuses).forEach(([productName, productState]) => { + if (productState.status === 'uninstalled') { + return; + } + const availableVersions = repositoryVersions[productName as ProductName]; + if (!availableVersions || !availableVersions.length) { + return; + } + const selectedVersion = selectVersion(this.currentVersion, availableVersions); + if (productState.version !== selectedVersion) { + toUpdate.push({ + productName: productName as ProductName, + productVersion: selectedVersion, + }); + } + }); + + for (const { productName, productVersion } of toUpdate) { + await this.installPackage({ + productName, + productVersion, + }); + } + } + + async installAll({}: {}) { + const repositoryVersions = await fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }); + const allProducts = Object.values(DocumentationProduct) as ProductName[]; + for (const productName of allProducts) { + const availableVersions = repositoryVersions[productName]; + if (!availableVersions || !availableVersions.length) { + this.log.warn(`No version found for product [${productName}]`); + continue; + } + const selectedVersion = selectVersion(this.currentVersion, availableVersions); + + await this.installPackage({ + productName, + productVersion: selectedVersion, + }); + } + } + + async installPackage({ + productName, + productVersion, + }: { + productName: ProductName; + productVersion: string; + }) { + this.log.info( + `Starting installing documentation for product [${productName}] and version [${productVersion}]` + ); + + productVersion = majorMinor(productVersion); + + await this.uninstallPackage({ productName }); + + let zipArchive: ZipArchive | undefined; + try { + await this.productDocClient.setInstallationStarted({ + productName, + productVersion, + }); + + await this.endpointManager.ensureInternalElserInstalled(); + + const artifactFileName = getArtifactName({ productName, productVersion }); + const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`; + const artifactPath = `${this.artifactsFolder}/${artifactFileName}`; + + this.log.debug(`Downloading from [${artifactUrl}] to [${artifactPath}]`); + await downloadToDisk(artifactUrl, artifactPath); + + zipArchive = await openZipArchive(artifactPath); + + validateArtifactArchive(zipArchive); + + const mappings = await loadMappingFile(zipArchive); + + const indexName = getProductDocIndexName(productName); + + await createIndex({ + indexName, + mappings, + esClient: this.esClient, + log: this.log, + }); + + await populateIndex({ + indexName, + archive: zipArchive, + esClient: this.esClient, + log: this.log, + }); + await this.productDocClient.setInstallationSuccessful(productName, indexName); + + this.log.info( + `Documentation installation successful for product [${productName}] and version [${productVersion}]` + ); + } catch (e) { + this.log.error( + `Error during documentation installation of product [${productName}]/[${productVersion}] : ${e.message}` + ); + + await this.productDocClient.setInstallationFailed(productName, e.message); + throw e; + } finally { + zipArchive?.close(); + } + } + + async uninstallPackage({ productName }: { productName: ProductName }) { + const indexName = getProductDocIndexName(productName); + await this.esClient.indices.delete( + { + index: indexName, + }, + { ignore: [404] } + ); + + await this.productDocClient.setUninstalled(productName); + } + + async uninstallAll() { + const allProducts = Object.values(DocumentationProduct); + for (const productName of allProducts) { + await this.uninstallPackage({ productName }); + } + } +} + +const selectVersion = (currentVersion: string, availableVersions: string[]): string => { + return availableVersions.includes(currentVersion) + ? currentVersion + : latestVersion(availableVersions); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts new file mode 100644 index 000000000000..fca8b5283c30 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { createIndex } from './create_index'; +import { internalElserInferenceId } from '../../../../common/consts'; + +describe('createIndex', () => { + let log: MockedLogger; + let esClient: ElasticsearchClient; + + beforeEach(() => { + log = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('calls esClient.indices.create with the right parameters', async () => { + const mappings: MappingTypeMapping = { + properties: {}, + }; + const indexName = '.some-index'; + + await createIndex({ + indexName, + mappings, + log, + esClient, + }); + + expect(esClient.indices.create).toHaveBeenCalledTimes(1); + expect(esClient.indices.create).toHaveBeenCalledWith({ + index: indexName, + mappings, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, + }); + }); + + it('rewrites the inference_id attribute of semantic_text fields in the mapping', async () => { + const mappings: MappingTypeMapping = { + properties: { + semantic: { + type: 'semantic_text', + inference_id: '.elser', + }, + bool: { + type: 'boolean', + }, + }, + }; + + await createIndex({ + indexName: '.some-index', + mappings, + log, + esClient, + }); + + expect(esClient.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ + mappings: { + properties: { + semantic: { + type: 'semantic_text', + inference_id: internalElserInferenceId, + }, + bool: { + type: 'boolean', + }, + }, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts new file mode 100644 index 000000000000..decd62e556ba --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { MappingTypeMapping, MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import { internalElserInferenceId } from '../../../../common/consts'; + +export const createIndex = async ({ + esClient, + indexName, + mappings, + log, +}: { + esClient: ElasticsearchClient; + indexName: string; + mappings: MappingTypeMapping; + log: Logger; +}) => { + log.debug(`Creating index ${indexName}`); + + overrideInferenceId(mappings, internalElserInferenceId); + + await esClient.indices.create({ + index: indexName, + mappings, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, + }); +}; + +const overrideInferenceId = (mappings: MappingTypeMapping, inferenceId: string) => { + const recursiveOverride = (current: MappingTypeMapping | MappingProperty) => { + if ('type' in current && current.type === 'semantic_text') { + current.inference_id = inferenceId; + } + if ('properties' in current && current.properties) { + for (const prop of Object.values(current.properties)) { + recursiveOverride(prop); + } + } + }; + recursiveOverride(mappings); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts new file mode 100644 index 000000000000..805008ccab69 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch, { Response } from 'node-fetch'; +import { fetchArtifactVersions } from './fetch_artifact_versions'; +import { getArtifactName, DocumentationProduct, ProductName } from '@kbn/product-doc-common'; + +jest.mock('node-fetch'); +const fetchMock = fetch as jest.MockedFn; + +const createResponse = ({ + artifactNames, + truncated = false, +}: { + artifactNames: string[]; + truncated?: boolean; +}) => { + return ` + + kibana-ai-assistant-kb-artifacts + + + ${truncated} + ${artifactNames.map( + (artifactName) => ` + + ${artifactName} + 1728486063097626 + 1 + 2024-10-09T15:01:03.137Z + "e0584955969eccf2a16b8829f768cb1f" + 36781438 + ` + )} + + `; +}; + +const artifactRepositoryUrl = 'https://lost.com'; + +const expectVersions = ( + versions: Partial> +): Record => { + const response = {} as Record; + Object.values(DocumentationProduct).forEach((productName) => { + response[productName] = []; + }); + return { + ...response, + ...versions, + }; +}; + +describe('fetchArtifactVersions', () => { + beforeEach(() => { + fetchMock.mockReset(); + }); + + const mockResponse = (responseText: string) => { + const response = { + text: () => Promise.resolve(responseText), + }; + fetchMock.mockResolvedValue(response as Response); + }; + + it('calls fetch with the right parameters', async () => { + mockResponse(createResponse({ artifactNames: [] })); + + await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith(`${artifactRepositoryUrl}?max-keys=1000`); + }); + + it('returns the list of versions from the repository', async () => { + const artifactNames = [ + getArtifactName({ productName: 'kibana', productVersion: '8.16' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }), + ]; + mockResponse(createResponse({ artifactNames })); + + const versions = await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(versions).toEqual( + expectVersions({ + kibana: ['8.16'], + elasticsearch: ['8.16'], + }) + ); + }); + + it('retrieve all versions for each product', async () => { + const artifactNames = [ + getArtifactName({ productName: 'kibana', productVersion: '8.15' }), + getArtifactName({ productName: 'kibana', productVersion: '8.16' }), + getArtifactName({ productName: 'kibana', productVersion: '8.17' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '9.0' }), + ]; + mockResponse(createResponse({ artifactNames })); + + const versions = await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(versions).toEqual( + expectVersions({ + kibana: ['8.15', '8.16', '8.17'], + elasticsearch: ['8.16', '9.0'], + }) + ); + }); + + it('throws an error if the response is truncated', async () => { + mockResponse(createResponse({ artifactNames: [], truncated: true })); + + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects.toThrowError( + /bucket content is truncated/ + ); + }); + + it('throws an error if the response is not valid xml', async () => { + mockResponse('some plain text'); + + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects.toThrowError(); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts new file mode 100644 index 000000000000..69c6db2d5d8a --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch from 'node-fetch'; +import { parseString } from 'xml2js'; +import { type ProductName, DocumentationProduct, parseArtifactName } from '@kbn/product-doc-common'; + +type ArtifactAvailableVersions = Record; + +export const fetchArtifactVersions = async ({ + artifactRepositoryUrl, +}: { + artifactRepositoryUrl: string; +}): Promise => { + const res = await fetch(`${artifactRepositoryUrl}?max-keys=1000`); + const xml = await res.text(); + return new Promise((resolve, reject) => { + parseString(xml, (err, result: ListBucketResponse) => { + if (err) { + reject(err); + } + + // 6 artifacts per minor stack version means we have a few decades before facing this problem + if (result.ListBucketResult.IsTruncated?.includes('true')) { + throw new Error('bucket content is truncated, cannot retrieve all versions'); + } + + const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); + + const record: ArtifactAvailableVersions = {} as ArtifactAvailableVersions; + allowedProductNames.forEach((product) => { + record[product] = []; + }); + + result.ListBucketResult.Contents?.forEach((contentEntry) => { + const artifactName = contentEntry.Key[0]; + const parsed = parseArtifactName(artifactName); + if (parsed) { + const { productName, productVersion } = parsed; + record[productName]!.push(productVersion); + } + }); + + resolve(record); + }); + }); +}; + +interface ListBucketResponse { + ListBucketResult: { + Name?: string[]; + IsTruncated?: string[]; + Contents?: Array<{ Key: string[] }>; + }; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts new file mode 100644 index 000000000000..3c84fc9cccf1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { createIndex } from './create_index'; +export { populateIndex } from './populate_index'; +export { validateArtifactArchive } from './validate_artifact_archive'; +export { fetchArtifactVersions } from './fetch_artifact_versions'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts new file mode 100644 index 000000000000..2f301f9928e9 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { times } from 'lodash'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { internalElserInferenceId } from '../../../../common/consts'; +import type { ZipArchive } from '../utils/zip_archive'; +import { populateIndex } from './populate_index'; + +const createMockArchive = (entries: Record): ZipArchive => { + return { + hasEntry: (entryPath) => Object.keys(entries).includes(entryPath), + getEntryPaths: () => Object.keys(entries), + getEntryContent: async (entryPath) => Buffer.from(entries[entryPath]), + close: () => undefined, + }; +}; + +const createContentFile = (count: number, offset: number = 0): string => { + return times(count) + .map((i) => JSON.stringify({ idx: offset + i })) + .join('\n'); +}; + +describe('populateIndex', () => { + let log: MockedLogger; + let esClient: ReturnType; + + beforeEach(() => { + log = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('calls `esClient.bulk` once per content file', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': createContentFile(2), + 'content/content-1.ndjson': createContentFile(2), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(2); + }); + + it('calls `esClient.bulk` with the right payload', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': createContentFile(2), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + expect(esClient.bulk).toHaveBeenCalledWith({ + refresh: false, + operations: [ + { index: { _index: '.foo' } }, + { idx: 0 }, + { index: { _index: '.foo' } }, + { idx: 1 }, + ], + }); + }); + + it('rewrites the inference_id of semantic fields', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': JSON.stringify({ + semantic: { text: 'foo', inference: { inference_id: '.some-inference' } }, + }), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + expect(esClient.bulk).toHaveBeenCalledWith({ + refresh: false, + operations: [ + { index: { _index: '.foo' } }, + { + semantic: { + inference: { + inference_id: internalElserInferenceId, + }, + text: 'foo', + }, + }, + ], + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts new file mode 100644 index 000000000000..017757ca90b9 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { isArtifactContentFilePath } from '@kbn/product-doc-common'; +import { internalElserInferenceId } from '../../../../common/consts'; +import type { ZipArchive } from '../utils/zip_archive'; + +export const populateIndex = async ({ + esClient, + indexName, + archive, + log, +}: { + esClient: ElasticsearchClient; + indexName: string; + archive: ZipArchive; + log: Logger; +}) => { + log.debug(`Starting populating index ${indexName}`); + + const contentEntries = archive.getEntryPaths().filter(isArtifactContentFilePath); + + for (let i = 0; i < contentEntries.length; i++) { + const entryPath = contentEntries[i]; + log.debug(`Indexing content for entry ${entryPath}`); + const contentBuffer = await archive.getEntryContent(entryPath); + await indexContentFile({ indexName, esClient, contentBuffer }); + } + + log.debug(`Done populating index ${indexName}`); +}; + +const indexContentFile = async ({ + indexName, + contentBuffer, + esClient, +}: { + indexName: string; + contentBuffer: Buffer; + esClient: ElasticsearchClient; +}) => { + const fileContent = contentBuffer.toString('utf-8'); + const lines = fileContent.split('\n'); + + const documents = lines + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + return JSON.parse(line); + }) + .map((doc) => rewriteInferenceId(doc, internalElserInferenceId)); + + const operations = documents.reduce((ops, document) => { + ops!.push(...[{ index: { _index: indexName } }, document]); + return ops; + }, [] as BulkRequest['operations']); + + const response = await esClient.bulk({ + refresh: false, + operations, + }); + + if (response.errors) { + const error = response.items.find((item) => item.index?.error)?.index?.error ?? 'unknown error'; + throw new Error(`Error indexing documents: ${JSON.stringify(error)}`); + } +}; + +const rewriteInferenceId = (document: Record, inferenceId: string) => { + // we don't need to handle nested fields, we don't have any and won't. + Object.values(document).forEach((field) => { + if (field.inference) { + field.inference.inference_id = inferenceId; + } + }); + return document; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts new file mode 100644 index 000000000000..607277aaf346 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ZipArchive } from '../utils/zip_archive'; +import { validateArtifactArchive } from './validate_artifact_archive'; + +const createMockArchive = (entryPaths: string[]): ZipArchive => { + return { + hasEntry: (entryPath) => entryPaths.includes(entryPath), + getEntryPaths: () => entryPaths, + getEntryContent: () => { + throw new Error('non implemented'); + }, + close: () => undefined, + }; +}; + +describe('validateArtifactArchive', () => { + it('validates that the archive contains all the mandatory files', () => { + const archive = createMockArchive([ + 'manifest.json', + 'mappings.json', + 'content/content-1.ndjson', + ]); + + const validation = validateArtifactArchive(archive); + + expect(validation).toEqual({ valid: true }); + }); + + it('does not validate if the archive does not contain a manifest', () => { + const archive = createMockArchive(['something.txt']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "Manifest file not found", + "valid": false, + } + `); + }); + + it('does not validate if the archive does not contain mappings', () => { + const archive = createMockArchive(['manifest.json']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "Mapping file not found", + "valid": false, + } + `); + }); + + it('does not validate if the archive does not contain content files', () => { + const archive = createMockArchive(['manifest.json', 'mappings.json']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "No content files were found", + "valid": false, + } + `); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts new file mode 100644 index 000000000000..471d7c080c48 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArtifactContentFilePath } from '@kbn/product-doc-common'; +import type { ZipArchive } from '../utils/zip_archive'; + +type ValidationResult = { valid: true } | { valid: false; error: string }; + +export const validateArtifactArchive = (archive: ZipArchive): ValidationResult => { + if (!archive.hasEntry('manifest.json')) { + return { valid: false, error: 'Manifest file not found' }; + } + if (!archive.hasEntry('mappings.json')) { + return { valid: false, error: 'Mapping file not found' }; + } + if (!archive.getEntryPaths().some(isArtifactContentFilePath)) { + return { valid: false, error: 'No content files were found' }; + } + return { valid: true }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts new file mode 100644 index 000000000000..9d42be652d74 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ArtifactManifest } from '@kbn/product-doc-common'; +import type { ZipArchive } from './zip_archive'; +import { loadManifestFile, loadMappingFile } from './archive_accessors'; + +const createMockArchive = (entries: Record): ZipArchive => { + return { + hasEntry: (entryPath) => Object.keys(entries).includes(entryPath), + getEntryPaths: () => Object.keys(entries), + getEntryContent: async (entryPath) => Buffer.from(entries[entryPath]), + close: () => undefined, + }; +}; + +describe('loadManifestFile', () => { + it('parses the manifest from the archive', async () => { + const manifest: ArtifactManifest = { + formatVersion: '1.0.0', + productName: 'kibana', + productVersion: '8.16', + }; + const archive = createMockArchive({ 'manifest.json': JSON.stringify(manifest) }); + + const parsedManifest = await loadManifestFile(archive); + + expect(parsedManifest).toEqual(manifest); + }); + + it('throws if the archive does not contain the manifest', async () => { + const archive = createMockArchive({}); + + await expect(loadManifestFile(archive)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Could not load archive file: \\"manifest.json\\" not found in archive"` + ); + }); + + it('throws if the manifest cannot be parsed', async () => { + const archive = createMockArchive({ 'manifest.json': '{}}}{' }); + + await expect(loadManifestFile(archive)).rejects.toThrowError(); + }); +}); + +describe('loadMappingFile', () => { + it('parses the manifest from the archive', async () => { + const mappings: MappingTypeMapping = { + properties: { + foo: { type: 'text' }, + }, + }; + const archive = createMockArchive({ 'mappings.json': JSON.stringify(mappings) }); + + const parsedMappings = await loadMappingFile(archive); + + expect(parsedMappings).toEqual(mappings); + }); + + it('throws if the archive does not contain the manifest', async () => { + const archive = createMockArchive({}); + + await expect(loadMappingFile(archive)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Could not load archive file: \\"mappings.json\\" not found in archive"` + ); + }); + + it('throws if the manifest cannot be parsed', async () => { + const archive = createMockArchive({ 'mappings.json': '{}}}{' }); + + await expect(loadMappingFile(archive)).rejects.toThrowError(); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts new file mode 100644 index 000000000000..a4ec4f4418f3 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ArtifactManifest } from '@kbn/product-doc-common'; +import type { ZipArchive } from './zip_archive'; + +const manifestEntryPath = 'manifest.json'; +const mappingsEntryPath = 'mappings.json'; + +export const loadManifestFile = async (archive: ZipArchive): Promise => { + return await parseEntryContent(manifestEntryPath, archive); +}; + +export const loadMappingFile = async (archive: ZipArchive): Promise => { + return await parseEntryContent(mappingsEntryPath, archive); +}; + +const parseEntryContent = async (entryPath: string, archive: ZipArchive): Promise => { + if (!archive.hasEntry(entryPath)) { + throw new Error(`Could not load archive file: "${entryPath}" not found in archive`); + } + try { + const buffer = await archive.getEntryContent(entryPath); + return JSON.parse(buffer.toString('utf-8')); + } catch (e) { + throw new Error(`Could not parse archive file: ${e}`); + } +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts new file mode 100644 index 000000000000..ea5357792ef5 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createWriteStream } from 'fs'; +import { mkdir } from 'fs/promises'; +import Path from 'path'; +import fetch from 'node-fetch'; + +export const downloadToDisk = async (fileUrl: string, filePath: string) => { + const dirPath = Path.dirname(filePath); + await mkdir(dirPath, { recursive: true }); + const res = await fetch(fileUrl); + const fileStream = createWriteStream(filePath); + await new Promise((resolve, reject) => { + res.body.pipe(fileStream); + res.body.on('error', reject); + fileStream.on('finish', resolve); + }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts new file mode 100644 index 000000000000..a612a8c6e9f4 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { downloadToDisk } from './download'; +export { openZipArchive, type ZipArchive } from './zip_archive'; +export { loadManifestFile, loadMappingFile } from './archive_accessors'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts new file mode 100644 index 000000000000..9bc20f2eecdb --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { latestVersion, majorMinor } from './semver'; + +describe('majorMinor', () => { + it('returns the version in a {major.minor} format', () => { + expect(majorMinor('9.17.5')).toEqual('9.17'); + }); + it('ignores qualifiers', () => { + expect(majorMinor('10.42.9000-snap')).toEqual('10.42'); + }); + it('accepts {major.minor} format as input', () => { + expect(majorMinor('8.16')).toEqual('8.16'); + }); +}); + +describe('latestVersion', () => { + it('returns the highest version from the list', () => { + expect(latestVersion(['7.16.3', '8.1.4', '6.14.2'])).toEqual('8.1.4'); + }); + it('accepts versions in a {major.minor} format', () => { + expect(latestVersion(['9.16', '9.3'])).toEqual('9.16'); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts new file mode 100644 index 000000000000..b4e38215af90 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Semver from 'semver'; + +export const latestVersion = (versions: string[]): string => { + let latest: string = versions[0]; + for (let i = 1; i < versions.length; i++) { + const current = versions[i]; + if (Semver.gt(Semver.coerce(current)!, Semver.coerce(latest)!)) { + latest = current; + } + } + return latest; +}; + +export const majorMinor = (version: string): string => { + const parsed = Semver.coerce(version); + if (!parsed) { + throw new Error(`Not a valid semver version: [${version}]`); + } + return `${parsed.major}.${parsed.minor}`; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..fce195d2c4db26c8590ec97f45dc9dfc3fcf8346 GIT binary patch literal 800 zcmWIWW@h1H00EW;|H!p|Div%%HVAVu$S{6KKJgobc3FlUtVrDpZQZmq# zR2(K5B20>^@{a^M6%N>ezROE3E=f(%2YYRIHedQ4AV%{VssZbSK3(Sk8Uez>2m>JI z#3S4UGOQTWO)zx<-i%Cg%(y~b0_;X$*fK0>1Tm50fE5x47>-1khZ*+B=6S))gT@5V zJWx#FF%L5akjv#`n9z0fnhJgZ-fMKX{g6ueX eU>HITgM~aK)!;Ii6_m^vSb=aFP-_V&3K#%85|UK_ literal 0 HcmV?d00001 diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts new file mode 100644 index 000000000000..71cd5891c5e5 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import { openZipArchive, ZipArchive } from './zip_archive'; + +const ZIP_PATH = Path.resolve(__dirname, './test_data/test_archive_1.zip'); + +describe('ZipArchive', () => { + let archive: ZipArchive; + + beforeAll(async () => { + archive = await openZipArchive(ZIP_PATH); + }); + + afterAll(() => { + archive?.close(); + }); + + test('#getEntryPaths returns the path of all entries', () => { + expect(archive.getEntryPaths().sort()).toEqual([ + 'nested/', + 'nested/nested_1.txt', + 'text_1.txt', + 'text_2.txt', + 'text_3.txt', + ]); + }); + + test('#hasEntry returns true if the entry exists, false otherwise', () => { + expect(archive.hasEntry('nested/nested_1.txt')).toBe(true); + expect(archive.hasEntry('not_an_entry')).toBe(false); + }); + + test('#getEntryContent returns the content of the entry', async () => { + const buffer = await archive.getEntryContent('text_1.txt'); + expect(buffer.toString('utf-8')).toEqual('text_1'); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts new file mode 100644 index 000000000000..dbc4ec1b3e41 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import yauzl from 'yauzl'; + +export interface ZipArchive { + hasEntry(entryPath: string): boolean; + getEntryPaths(): string[]; + getEntryContent(entryPath: string): Promise; + close(): void; +} + +export const openZipArchive = async (archivePath: string): Promise => { + return new Promise((resolve, reject) => { + const entries: yauzl.Entry[] = []; + yauzl.open(archivePath, { lazyEntries: true, autoClose: false }, (err, zipFile) => { + if (err || !zipFile) { + return reject(err ?? 'No zip file'); + } + + zipFile!.on('entry', (entry) => { + entries.push(entry); + zipFile.readEntry(); + }); + + zipFile.on('end', () => { + const archive = new ZipArchiveImpl(entries, zipFile); + resolve(archive); + }); + + zipFile.on('close', () => {}); + + zipFile.readEntry(); + }); + }); +}; + +class ZipArchiveImpl implements ZipArchive { + private readonly zipFile: yauzl.ZipFile; + private readonly entries: Map; + + constructor(entries: yauzl.Entry[], zipFile: yauzl.ZipFile) { + this.zipFile = zipFile; + this.entries = new Map(entries.map((entry) => [entry.fileName, entry])); + } + + hasEntry(entryPath: string) { + return this.entries.has(entryPath); + } + + getEntryPaths() { + return [...this.entries.keys()]; + } + + getEntryContent(entryPath: string) { + const foundEntry = this.entries.get(entryPath); + if (!foundEntry) { + throw new Error(`Entry ${entryPath} not found in archive`); + } + return getZipEntryContent(this.zipFile, foundEntry); + } + + close() { + this.zipFile.close(); + } +} + +const getZipEntryContent = async (zipFile: yauzl.ZipFile, entry: yauzl.Entry): Promise => { + return new Promise((resolve, reject) => { + zipFile.openReadStream(entry, (err, readStream) => { + if (err) { + return reject(err); + } else { + const chunks: Buffer[] = []; + readStream!.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); + readStream!.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + readStream!.on('error', () => { + reject(); + }); + } + }); + }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts new file mode 100644 index 000000000000..3e5ac95ae4ed --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { SearchService } from './search_service'; +export type { DocSearchOptions, DocSearchResult, DocSearchResponse, SearchApi } from './types'; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts similarity index 78% rename from x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index 373a6b875542..03c3b72f86f9 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -5,29 +5,27 @@ * 2.0. */ -import type { Client } from '@elastic/elasticsearch'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; // https://search-labs.elastic.co/search-labs/blog/elser-rag-search-for-relevance -export const performSemanticSearch = async ({ +export const performSearch = async ({ searchQuery, + size, index, client, }: { searchQuery: string; - index: string; - client: Client; + size: number; + index: string | string[]; + client: ElasticsearchClient; }) => { - const results = await client.search({ + const results = await client.search({ index, - size: 3, + size, query: { bool: { - filter: { - bool: { - must: [{ term: { version: '8.15' } }], - }, - }, should: [ { multi_match: { @@ -37,7 +35,7 @@ export const performSemanticSearch = async ({ fields: [ 'content_title', 'content_body.text', - 'ai_subtitle.text', + 'ai_subtitle', 'ai_summary.text', 'ai_questions_answered.text', 'ai_tags', @@ -65,12 +63,6 @@ export const performSemanticSearch = async ({ query: searchQuery, }, }, - { - semantic: { - field: 'ai_subtitle', - query: searchQuery, - }, - }, { semantic: { field: 'ai_summary', diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts new file mode 100644 index 000000000000..c8053ca981e7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { SearchService } from './search_service'; +import { getIndicesForProductNames } from './utils'; + +import { performSearch } from './perform_search'; +jest.mock('./perform_search'); +const performSearchMock = performSearch as jest.MockedFn; + +describe('SearchService', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let service: SearchService; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + service = new SearchService({ logger, esClient }); + + performSearchMock.mockResolvedValue([]); + }); + + afterEach(() => { + performSearchMock.mockReset(); + }); + + describe('#search', () => { + it('calls `performSearch` with the right parameters', async () => { + await service.search({ + query: 'What is Kibana?', + products: ['kibana'], + max: 42, + }); + + expect(performSearchMock).toHaveBeenCalledTimes(1); + expect(performSearchMock).toHaveBeenCalledWith({ + searchQuery: 'What is Kibana?', + size: 42, + index: getIndicesForProductNames(['kibana']), + client: esClient, + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts new file mode 100644 index 000000000000..a0b1e4fd4a83 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { getIndicesForProductNames, mapResult } from './utils'; +import { performSearch } from './perform_search'; +import type { DocSearchOptions, DocSearchResponse } from './types'; + +export class SearchService { + private readonly log: Logger; + private readonly esClient: ElasticsearchClient; + + constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) { + this.log = logger; + this.esClient = esClient; + } + + async search(options: DocSearchOptions): Promise { + const { query, max = 3, products } = options; + this.log.debug(`performing search - query=[${query}]`); + const results = await performSearch({ + searchQuery: query, + size: max, + index: getIndicesForProductNames(products), + client: this.esClient, + }); + + return { + results: results.map(mapResult), + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts new file mode 100644 index 000000000000..fb474bbf4dea --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; + +export interface DocSearchOptions { + query: string; + max?: number; + products?: ProductName[]; +} + +export interface DocSearchResult { + title: string; + content: string; + url: string; + productName: ProductName; +} + +export interface DocSearchResponse { + results: DocSearchResult[]; +} + +export type SearchApi = (options: DocSearchOptions) => Promise; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts new file mode 100644 index 000000000000..0293d086d4f1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { productDocIndexPattern, getProductDocIndexName } from '@kbn/product-doc-common'; +import { getIndicesForProductNames } from './get_indices_for_product_names'; + +describe('getIndicesForProductNames', () => { + it('returns the index pattern when product names are not specified', () => { + expect(getIndicesForProductNames(undefined)).toEqual(productDocIndexPattern); + expect(getIndicesForProductNames([])).toEqual(productDocIndexPattern); + }); + it('returns individual index names when product names are specified', () => { + expect(getIndicesForProductNames(['kibana', 'elasticsearch'])).toEqual([ + getProductDocIndexName('kibana'), + getProductDocIndexName('elasticsearch'), + ]); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts new file mode 100644 index 000000000000..e97ed9cea361 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + productDocIndexPattern, + getProductDocIndexName, + type ProductName, +} from '@kbn/product-doc-common'; + +export const getIndicesForProductNames = ( + productNames: ProductName[] | undefined +): string | string[] => { + if (!productNames || !productNames.length) { + return productDocIndexPattern; + } + return productNames.map(getProductDocIndexName); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts new file mode 100644 index 000000000000..1a6a2eaa24a9 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getIndicesForProductNames } from './get_indices_for_product_names'; +export { mapResult } from './map_result'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts new file mode 100644 index 000000000000..56e8ce4875cc --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; +import { mapResult } from './map_result'; + +const createHit = ( + attrs: ProductDocumentationAttributes +): SearchHit => { + return { + _index: '.foo', + _source: attrs, + }; +}; + +describe('mapResult', () => { + it('returns the expected shape', () => { + const input = createHit({ + content_title: 'content_title', + content_body: { text: 'content_body' }, + product_name: 'kibana', + root_type: 'documentation', + slug: 'foo.html', + url: 'http://lost.com/foo.html', + version: '8.16', + ai_subtitle: 'ai_subtitle', + ai_summary: { text: 'ai_summary' }, + ai_questions_answered: { text: ['question A'] }, + ai_tags: ['foo', 'bar', 'test'], + }); + + const output = mapResult(input); + + expect(output).toEqual({ + content: 'content_body', + productName: 'kibana', + title: 'content_title', + url: 'http://lost.com/foo.html', + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts new file mode 100644 index 000000000000..f4f66b211182 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; +import type { DocSearchResult } from '../types'; + +export const mapResult = (docHit: SearchHit): DocSearchResult => { + return { + title: docHit._source!.content_title, + content: docHit._source!.content_body.text, + url: docHit._source!.url, + productName: docHit._source!.product_name, + }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts new file mode 100644 index 000000000000..d971561914ff --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const ENSURE_DOC_UP_TO_DATE_TASK_TYPE = 'ProductDocBase:EnsureUpToDate'; +export const ENSURE_DOC_UP_TO_DATE_TASK_ID = 'ProductDocBase:EnsureUpToDate'; + +export const registerEnsureUpToDateTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [ENSURE_DOC_UP_TO_DATE_TASK_TYPE]: { + title: 'Ensure product documentation up to date task', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.ensureUpToDate({}); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleEnsureUpToDateTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: ENSURE_DOC_UP_TO_DATE_TASK_ID, + taskType: ENSURE_DOC_UP_TO_DATE_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(ENSURE_DOC_UP_TO_DATE_TASK_ID); + + logger.info(`Task ${ENSURE_DOC_UP_TO_DATE_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return ENSURE_DOC_UP_TO_DATE_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts new file mode 100644 index 000000000000..0b5833055fd8 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { registerEnsureUpToDateTaskDefinition } from './ensure_up_to_date'; +import { registerInstallAllTaskDefinition } from './install_all'; +import { registerUninstallAllTaskDefinition } from './uninstall_all'; + +export const registerTaskDefinitions = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + registerEnsureUpToDateTaskDefinition({ getServices, taskManager }); + registerInstallAllTaskDefinition({ getServices, taskManager }); + registerUninstallAllTaskDefinition({ getServices, taskManager }); +}; + +export { scheduleEnsureUpToDateTask, ENSURE_DOC_UP_TO_DATE_TASK_ID } from './ensure_up_to_date'; +export { scheduleInstallAllTask, INSTALL_ALL_TASK_ID } from './install_all'; +export { scheduleUninstallAllTask, UNINSTALL_ALL_TASK_ID } from './uninstall_all'; +export { waitUntilTaskCompleted, getTaskStatus } from './utils'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts new file mode 100644 index 000000000000..0d2cc48fb06b --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const INSTALL_ALL_TASK_TYPE = 'ProductDocBase:InstallAll'; +export const INSTALL_ALL_TASK_ID = 'ProductDocBase:InstallAll'; + +export const registerInstallAllTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [INSTALL_ALL_TASK_TYPE]: { + title: 'Install all product documentation artifacts', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.installAll({}); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleInstallAllTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: INSTALL_ALL_TASK_ID, + taskType: INSTALL_ALL_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(INSTALL_ALL_TASK_ID); + + logger.info(`Task ${INSTALL_ALL_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return INSTALL_ALL_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts new file mode 100644 index 000000000000..6a88fec205dd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const UNINSTALL_ALL_TASK_TYPE = 'ProductDocBase:UninstallAll'; +export const UNINSTALL_ALL_TASK_ID = 'ProductDocBase:UninstallAll'; + +export const registerUninstallAllTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [UNINSTALL_ALL_TASK_TYPE]: { + title: 'Uninstall all product documentation artifacts', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.uninstallAll(); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleUninstallAllTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: UNINSTALL_ALL_TASK_ID, + taskType: UNINSTALL_ALL_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(UNINSTALL_ALL_TASK_ID); + + logger.info(`Task ${UNINSTALL_ALL_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return UNINSTALL_ALL_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts new file mode 100644 index 000000000000..e32ea02a11b0 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; + +export const getTaskStatus = async ({ + taskManager, + taskId, +}: { + taskManager: TaskManagerStartContract; + taskId: string; +}) => { + try { + const taskInstance = await taskManager.get(taskId); + return taskInstance.status; + } catch (e) { + // not found means the task was completed and the entry removed + if (SavedObjectsErrorHelpers.isNotFoundError(e)) { + return 'not_scheduled'; + } + throw e; + } +}; + +export const isTaskCurrentlyRunningError = (err: Error): boolean => { + return err.message?.includes('currently running'); +}; + +export const waitUntilTaskCompleted = async ({ + taskManager, + taskId, + timeout = 120_000, + interval = 5_000, +}: { + taskManager: TaskManagerStartContract; + taskId: string; + timeout?: number; + interval?: number; +}): Promise => { + const start = Date.now(); + const max = start + timeout; + let now = start; + while (now < max) { + try { + const taskInstance = await taskManager.get(taskId); + const { status } = taskInstance; + if (status === 'idle' || status === 'claiming' || status === 'running') { + await sleep(interval); + now = Date.now(); + } else { + return; + } + } catch (e) { + if (SavedObjectsErrorHelpers.isNotFoundError(e)) { + // not found means the task was completed and the entry removed + return; + } + } + } + + throw new Error(`Timeout waiting for task ${taskId} to complete.`); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts new file mode 100644 index 000000000000..f00943b69670 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { SearchApi } from './services/search'; +import type { ProductDocInstallClient } from './services/doc_install_status'; +import type { PackageInstaller } from './services/package_installer'; +import type { DocumentationManager, DocumentationManagerAPI } from './services/doc_manager'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface ProductDocBaseSetupDependencies { + taskManager: TaskManagerSetupContract; +} + +export interface ProductDocBaseStartDependencies { + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} + +export interface ProductDocBaseSetupContract {} + +export interface ProductDocBaseStartContract { + search: SearchApi; + management: DocumentationManagerAPI; +} + +export interface InternalServices { + logger: Logger; + installClient: ProductDocInstallClient; + packageInstaller: PackageInstaller; + documentationManager: DocumentationManager; + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json new file mode 100644 index 000000000000..9a2d1969556b --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -0,0 +1,29 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + "scripts/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/config-schema", + "@kbn/product-doc-common", + "@kbn/core-saved-objects-server", + "@kbn/utils", + "@kbn/core-http-browser", + "@kbn/logging-mocks", + "@kbn/licensing-plugin", + "@kbn/task-manager-plugin", + ] +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index 361d13e6d77f..f693fa53c06c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -77,7 +77,7 @@ export class ObservabilityAIAssistantPlugin privileges: { all: { app: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'kibana'], - api: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'ai_assistant'], + api: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'ai_assistant', 'manage_llm_product_doc'], catalogue: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID], savedObject: { all: [ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc index efc948503b0c..957ca0272c08 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc @@ -32,7 +32,8 @@ "alerting", "features", "inference", - "logsDataAccess" + "logsDataAccess", + "llmTasks" ], "optionalPlugins": [ "cloud" diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts new file mode 100644 index 000000000000..00072e0c79c4 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DocumentationProduct } from '@kbn/product-doc-common'; +import { FunctionVisibility } from '@kbn/observability-ai-assistant-plugin/common'; +import type { FunctionRegistrationParameters } from '.'; + +export const RETRIEVE_DOCUMENTATION_NAME = 'retrieve_elastic_doc'; + +export async function registerDocumentationFunction({ + functions, + resources, + pluginsStart: { llmTasks }, +}: FunctionRegistrationParameters) { + const isProductDocAvailable = (await llmTasks.retrieveDocumentationAvailable()) ?? false; + + functions.registerInstruction(({ availableFunctionNames }) => { + return availableFunctionNames.includes(RETRIEVE_DOCUMENTATION_NAME) + ? `When asked questions about the Elastic stack or products, You should use the ${RETRIEVE_DOCUMENTATION_NAME} function before answering, + to retrieve documentation related to the question. Consider that the documentation returned by the function + is always more up to date and accurate than any own internal knowledge you might have.` + : undefined; + }); + + functions.registerFunction( + { + name: RETRIEVE_DOCUMENTATION_NAME, + visibility: isProductDocAvailable + ? FunctionVisibility.AssistantOnly + : FunctionVisibility.Internal, + description: `Use this function to retrieve documentation about Elastic products. + You can retrieve documentation about the Elastic stack, such as Kibana and Elasticsearch, + or for Elastic solutions, such as Elastic Security, Elastic Observability or Elastic Enterprise Search + `, + parameters: { + type: 'object', + properties: { + query: { + description: `The query to use to retrieve documentation + Examples: + - "How to enable TLS for Elasticsearch?" + - "What is Kibana Lens?"`, + type: 'string' as const, + }, + product: { + description: `If specified, will filter the products to retrieve documentation for + Possible options are: + - "kibana": Kibana product + - "elasticsearch": Elasticsearch product + - "observability": Elastic Observability solution + - "security": Elastic Security solution + If not specified, will search against all products + `, + type: 'string' as const, + enum: Object.values(DocumentationProduct), + }, + }, + required: ['query'], + } as const, + }, + async ({ arguments: { query, product }, connectorId, useSimulatedFunctionCalling }) => { + const response = await llmTasks!.retrieveDocumentation({ + searchTerm: query, + products: product ? [product] : undefined, + max: 3, + connectorId, + request: resources.request, + functionCalling: useSimulatedFunctionCalling ? 'simulated' : 'native', + }); + + return { + content: { + documents: response.documents, + }, + }; + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts index 7554164a55a6..ba876ad9457b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts @@ -12,6 +12,7 @@ import { registerLensFunction } from './lens'; import { registerVisualizeESQLFunction } from './visualize_esql'; import { ObservabilityAIAssistantAppPluginStartDependencies } from '../types'; import { registerChangesFunction } from './changes'; +import { registerDocumentationFunction } from './documentation'; export type FunctionRegistrationParameters = Omit< Parameters[0], @@ -24,4 +25,5 @@ export const registerFunctions = async (registrationParameters: FunctionRegistra registerVisualizeESQLFunction(registrationParameters); registerAlertsFunction(registrationParameters); registerChangesFunction(registrationParameters); + await registerDocumentationFunction(registrationParameters); }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts index fc39e0b7fb24..a1196be6a829 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts @@ -37,6 +37,7 @@ import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plu import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import type { InferenceServerStart, InferenceServerSetup } from '@kbn/inference-plugin/server'; import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/server'; +import type { LlmTasksPluginStart } from '@kbn/llm-tasks-plugin/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ObservabilityAIAssistantAppServerStart {} @@ -57,6 +58,7 @@ export interface ObservabilityAIAssistantAppPluginStartDependencies { serverless?: ServerlessPluginStart; inference: InferenceServerStart; logsDataAccess: LogsDataAccessPluginStart; + llmTasks: LlmTasksPluginStart; } export interface ObservabilityAIAssistantAppPluginSetupDependencies { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json index 6608799caaf6..e0a520fb574c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json @@ -70,6 +70,8 @@ "@kbn/logs-data-access-plugin", "@kbn/ai-assistant-common", "@kbn/inference-common", + "@kbn/llm-tasks-plugin", + "@kbn/product-doc-common", ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc index cda6fdf0192f..c228f147dbfc 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc @@ -21,10 +21,11 @@ "optionalPlugins": [ "home", "serverless", + "productDocBase" ], "requiredBundles": [ "kibanaReact", - "logsDataAccess", + "logsDataAccess" ] } } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts index a680da5ed3f9..3bfe3dff3f9f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts @@ -9,6 +9,9 @@ export const REACT_QUERY_KEYS = { GET_GENAI_CONNECTORS: 'get_genai_connectors', GET_KB_ENTRIES: 'get_kb_entries', GET_KB_USER_INSTRUCTIONS: 'get_kb_user_instructions', + GET_PRODUCT_DOC_STATUS: 'get_product_doc_status', + INSTALL_PRODUCT_DOC: 'install_product_doc', + UNINSTALL_PRODUCT_DOC: 'uninstall_product_doc', CREATE_KB_ENTRIES: 'create_kb_entry', IMPORT_KB_ENTRIES: 'import_kb_entry', }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts new file mode 100644 index 000000000000..ef95d51f78d4 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from '@tanstack/react-query'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +export function useGetProductDocStatus() { + const { productDocBase } = useKibana().services; + + const { isLoading, isError, isSuccess, isRefetching, data, refetch } = useQuery({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + queryFn: async () => { + return productDocBase!.installation.getStatus(); + }, + keepPreviousData: false, + refetchOnWindowFocus: false, + }); + + return { + status: data, + refetch, + isLoading, + isRefetching, + isSuccess, + isError, + }; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts new file mode 100644 index 000000000000..cb32efa7e390 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import type { PerformInstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +type ServerError = IHttpFetchError; + +export function useInstallProductDoc() { + const { + productDocBase, + notifications: { toasts }, + } = useKibana().services; + const queryClient = useQueryClient(); + + return useMutation( + [REACT_QUERY_KEYS.INSTALL_PRODUCT_DOC], + () => { + return productDocBase!.installation.install(); + }, + { + onSuccess: () => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.successNotification', + { + defaultMessage: 'The Elastic documentation was successfully installed', + } + ) + ); + + queryClient.invalidateQueries({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + refetchType: 'all', + }); + }, + onError: (error) => { + toasts.addError(new Error(error.body?.message ?? error.message), { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.errorNotification', + { + defaultMessage: 'Something went wrong while installing the Elastic documentation', + } + ), + }); + }, + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts new file mode 100644 index 000000000000..4aa3b5423faa --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import type { UninstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +type ServerError = IHttpFetchError; + +export function useUninstallProductDoc() { + const { + productDocBase, + notifications: { toasts }, + } = useKibana().services; + const queryClient = useQueryClient(); + + return useMutation( + [REACT_QUERY_KEYS.UNINSTALL_PRODUCT_DOC], + () => { + return productDocBase!.installation.uninstall(); + }, + { + onSuccess: () => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.successNotification', + { + defaultMessage: 'The Elastic documentation was successfully uninstalled', + } + ) + ); + + queryClient.invalidateQueries({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + refetchType: 'all', + }); + }, + onError: (error) => { + toasts.addError(new Error(error.body?.message ?? error.message), { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.errorNotification', + { + defaultMessage: 'Something went wrong while uninstalling the Elastic documentation', + } + ), + }); + }, + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts index b7c6bb089663..67b294a5fef3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts @@ -10,6 +10,7 @@ import type { CoreSetup, Plugin, PluginInitializerContext } from '@kbn/core/publ import type { ManagementSetup } from '@kbn/management-plugin/public'; import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; import type { ServerlessPluginStart } from '@kbn/serverless/public'; +import type { ProductDocBasePluginStart } from '@kbn/product-doc-base-plugin/public'; import type { ObservabilityAIAssistantPublicSetup, @@ -31,6 +32,7 @@ export interface SetupDependencies { export interface StartDependencies { observabilityAIAssistant: ObservabilityAIAssistantPublicStart; serverless?: ServerlessPluginStart; + productDocBase?: ProductDocBasePluginStart; } export interface ConfigSchema { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx new file mode 100644 index 000000000000..668e363d071e --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState, useCallback, useMemo } from 'react'; +import { + EuiButton, + EuiDescribedFormGroup, + EuiFormRow, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiHealth, + EuiLoadingSpinner, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useKibana } from '../../../hooks/use_kibana'; +import { useGetProductDocStatus } from '../../../hooks/use_get_product_doc_status'; +import { useInstallProductDoc } from '../../../hooks/use_install_product_doc'; +import { useUninstallProductDoc } from '../../../hooks/use_uninstall_product_doc'; + +export function ProductDocEntry() { + const { overlays } = useKibana().services; + + const [isInstalled, setInstalled] = useState(true); + const [isInstalling, setInstalling] = useState(false); + + const { mutateAsync: installProductDoc } = useInstallProductDoc(); + const { mutateAsync: uninstallProductDoc } = useUninstallProductDoc(); + const { status, isLoading: isStatusLoading } = useGetProductDocStatus(); + + useEffect(() => { + if (status) { + setInstalled(status.overall === 'installed'); + } + }, [status]); + + const onClickInstall = useCallback(() => { + setInstalling(true); + installProductDoc().then( + () => { + setInstalling(false); + setInstalled(true); + }, + () => { + setInstalling(false); + setInstalled(false); + } + ); + }, [installProductDoc]); + + const onClickUninstall = useCallback(() => { + overlays + .openConfirm( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocUninstallConfirmText', + { + defaultMessage: `Are you sure you want to uninstall the Elastic documentation?`, + } + ), + { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocUninstallConfirmTitle', + { + defaultMessage: `Uninstalling Elastic documentation`, + } + ), + } + ) + .then((confirmed) => { + if (confirmed) { + uninstallProductDoc().then(() => { + setInstalling(false); + setInstalled(false); + }); + } + }); + }, [overlays, uninstallProductDoc]); + + const content = useMemo(() => { + if (isStatusLoading) { + return <>; + } + if (isInstalling) { + return ( + + + + + + + ); + } + if (isInstalled) { + return ( + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.installProductDocInstalledLabel', + { defaultMessage: 'Installed' } + )} + + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.uninstallProductDocButtonLabel', + { defaultMessage: 'Uninstall' } + )} + + + + ); + } + return ( + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.installProductDocButtonLabel', + { defaultMessage: 'Install' } + )} + + + + ); + }, [isInstalled, isInstalling, isStatusLoading, onClickInstall, onClickUninstall]); + + return ( + + {i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.productDocLabel', { + defaultMessage: 'Elastic documentation', + })} + + } + description={ +

+ + {i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.techPreview', { + defaultMessage: '[technical preview] ', + })} + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocDescription', + { + defaultMessage: + "Install Elastic documentation to improve the assistant's efficiency.", + } + )} +

+ } + > + {content} + + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx index 831ba9ff5805..00c3fb76ae66 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx @@ -11,10 +11,12 @@ import { i18n } from '@kbn/i18n'; import { useAppContext } from '../../../hooks/use_app_context'; import { useKibana } from '../../../hooks/use_kibana'; import { UISettings } from './ui_settings'; +import { ProductDocEntry } from './product_doc_entry'; export function SettingsTab() { const { application: { navigateToApp }, + productDocBase, } = useKibana().services; const { config } = useAppContext(); @@ -108,6 +110,7 @@ export function SettingsTab() { + {productDocBase ? : undefined} ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json index bc5cf69357dc..7b78d52c6480 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json @@ -26,7 +26,8 @@ "@kbn/logs-data-access-plugin", "@kbn/core-plugins-browser", "@kbn/ai-assistant", - "@kbn/core-plugins-server" + "@kbn/core-plugins-server", + "@kbn/product-doc-base-plugin" ], "exclude": [ "target/**/*" diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 88ef256b353e..a6bf7e7e9d5f 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -49,6 +49,9 @@ export default function ({ getService }: FtrProviderContext) { 'Fleet-Usage-Logger', 'Fleet-Usage-Sender', 'ML:saved-objects-sync', + 'ProductDocBase:EnsureUpToDate', + 'ProductDocBase:InstallAll', + 'ProductDocBase:UninstallAll', 'SLO:ORPHAN_SUMMARIES-CLEANUP-TASK', 'Synthetics:Clean-Up-Package-Policies', 'UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects', diff --git a/yarn.lock b/yarn.lock index 79da7243edd6..18b5fed51c27 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5550,6 +5550,10 @@ version "0.0.0" uid "" +"@kbn/llm-tasks-plugin@link:x-pack/plugins/ai_infra/llm_tasks": + version "0.0.0" + uid "" + "@kbn/locator-examples-plugin@link:examples/locator_examples": version "0.0.0" uid "" @@ -6034,6 +6038,14 @@ version "0.0.0" uid "" +"@kbn/product-doc-base-plugin@link:x-pack/plugins/ai_infra/product_doc_base": + version "0.0.0" + uid "" + +"@kbn/product-doc-common@link:x-pack/packages/ai-infra/product-doc-common": + version "0.0.0" + uid "" + "@kbn/profiling-data-access-plugin@link:x-pack/plugins/observability_solution/profiling_data_access": version "0.0.0" uid ""