Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[LLM tasks] Add product documentation retrieval task #194379

Merged
merged 113 commits into from
Nov 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
113 commits
Select commit Hold shift + click to select a range
76e3984
Create knowledge base registry plugin
pgayvallet Sep 30, 2024
89e033c
run bootstrap
pgayvallet Sep 30, 2024
b3c67ee
[CI] Auto-commit changed files from 'node scripts/generate codeowners'
kibanamachine Sep 30, 2024
3dc368e
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Sep 30, 2024
54430dd
Add knowledge_base_entry SO type
pgayvallet Sep 30, 2024
7694e75
make this empty plugin work, maybe?
pgayvallet Oct 1, 2024
e7c482f
fix SO meta
pgayvallet Oct 1, 2024
763dad5
move attributes to server folder
pgayvallet Oct 1, 2024
596b036
some tsdoc
pgayvallet Oct 1, 2024
b3097df
[CI] Auto-commit changed files from 'node scripts/check_mappings_upda…
kibanamachine Oct 1, 2024
dac2b40
[CI] Auto-commit changed files from 'node scripts/jest_integration -u…
kibanamachine Oct 1, 2024
65cc3e7
add new type to list
pgayvallet Oct 2, 2024
586e30f
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 2, 2024
febae38
installer: WIP
pgayvallet Oct 3, 2024
1f831e1
add plugin config
pgayvallet Oct 3, 2024
b2e3c49
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 4, 2024
5292ad2
add inference endpoint installation logic
pgayvallet Oct 4, 2024
0689dbc
reports install status
pgayvallet Oct 4, 2024
a9f085a
Add bucket content parsing
pgayvallet Oct 7, 2024
bfc8e6a
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 7, 2024
8bbd7e8
create new package
pgayvallet Oct 7, 2024
8e96808
factorize reusable bits into common package
pgayvallet Oct 7, 2024
8c0da9d
rename plugin
pgayvallet Oct 7, 2024
7ba6ee3
commit autochange
pgayvallet Oct 7, 2024
3f16f27
[CI] Auto-commit changed files from 'node scripts/build_plugin_list_d…
kibanamachine Oct 7, 2024
af0ee26
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Oct 7, 2024
96e5df5
move things
pgayvallet Oct 7, 2024
3ecaa5b
[CI] Auto-commit changed files from 'node scripts/yarn_deduplicate'
kibanamachine Oct 7, 2024
0369210
add first version of search
pgayvallet Oct 8, 2024
7cfb398
tweaks in artifact builder
pgayvallet Oct 8, 2024
258ffc9
add package upgrade check
pgayvallet Oct 9, 2024
63e7597
add llmTasks plugin
pgayvallet Oct 9, 2024
b039828
Add product documentation installation from the o11y assistant config
pgayvallet Oct 11, 2024
75aaaf6
add task availability check, improve summarizer
pgayvallet Oct 14, 2024
7b88930
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 14, 2024
cc727d2
[CI] Auto-commit changed files from 'node scripts/generate codeowners'
kibanamachine Oct 14, 2024
0e6e0d6
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Oct 14, 2024
1f11493
[CI] Auto-commit changed files from 'node scripts/yarn_deduplicate'
kibanamachine Oct 14, 2024
e4d9f9b
working on config page
pgayvallet Oct 14, 2024
2deea8b
revert old mappings
pgayvallet Oct 14, 2024
55bf175
update limits
pgayvallet Oct 14, 2024
084c78c
rename things
pgayvallet Oct 14, 2024
33fd538
fix imports
pgayvallet Oct 14, 2024
23bb7fd
fix jest file
pgayvallet Oct 14, 2024
05bb3b9
remove type that never existed ever
pgayvallet Oct 14, 2024
c1c01a9
silence i18n check's output
delanni Oct 14, 2024
ed89047
Merge branch 'main' into kbn-193849-add-kb-registry-plugin
delanni Oct 14, 2024
25d6132
[CI] Auto-commit changed files from 'node scripts/check_mappings_upda…
kibanamachine Oct 14, 2024
f67405e
[CI] Auto-commit changed files from 'node scripts/eslint --no-cache -…
kibanamachine Oct 14, 2024
0b13919
[CI] Auto-commit changed files from 'node scripts/jest_integration -u…
kibanamachine Oct 14, 2024
81c4ddf
set logging to quiet in i18n.sh, still log the error in case somethin…
delanni Oct 15, 2024
2fcfdee
self-review, first batch
pgayvallet Oct 14, 2024
0059e66
adding tests
pgayvallet Oct 15, 2024
7448517
more tests and fixes
pgayvallet Oct 16, 2024
77caf77
more tests and fixes bis
pgayvallet Oct 16, 2024
6a15d09
almost done
pgayvallet Oct 16, 2024
d3624d0
last batch of unit tests
pgayvallet Oct 17, 2024
b5116e3
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 17, 2024
b0ac940
[CI] Auto-commit changed files from 'node scripts/build_plugin_list_d…
kibanamachine Oct 17, 2024
5797829
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Oct 17, 2024
904416e
[CI] Auto-commit changed files from 'node scripts/yarn_deduplicate'
kibanamachine Oct 17, 2024
2c51c37
remove bundle from limits
pgayvallet Oct 17, 2024
3c79e40
fix failures
pgayvallet Oct 17, 2024
83709a8
lint
pgayvallet Oct 17, 2024
0382e2b
add endpoint access control
pgayvallet Oct 17, 2024
63a0e6b
add basic document processing to improve content
pgayvallet Oct 18, 2024
52ccfa9
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 18, 2024
9e96d08
bump summary threshold to 1k
pgayvallet Oct 18, 2024
9807a21
improve summary prompt
pgayvallet Oct 18, 2024
4f23984
change comment
pgayvallet Oct 18, 2024
68978d9
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 23, 2024
ef794c3
remove empty filter
kertal Oct 23, 2024
dec6903
address review comments on i18n script changes
pgayvallet Oct 23, 2024
2425274
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 29, 2024
b62b18b
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Oct 30, 2024
c4824f6
add license check for doc installation
pgayvallet Oct 30, 2024
30ae935
add timeout on endpoint creation
pgayvallet Oct 30, 2024
5ebcef4
fix tool registration due to API change
pgayvallet Oct 30, 2024
64648b0
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Oct 30, 2024
6306a52
fix plugin tests
pgayvallet Oct 30, 2024
1fe4de7
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Nov 7, 2024
3c13779
[CI] Auto-commit changed files from 'node scripts/generate codeowners'
kibanamachine Nov 7, 2024
c85e9cc
fix types for new inference package
pgayvallet Nov 7, 2024
0f0a494
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Nov 7, 2024
d7dc1bc
add token / reduction parameters
pgayvallet Nov 7, 2024
17d089c
Address review nits
pgayvallet Nov 11, 2024
2dc68c2
more review nits
pgayvallet Nov 11, 2024
02f286e
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Nov 11, 2024
e31a913
Add instructions for the retrieve doc function
pgayvallet Nov 11, 2024
57ed9d4
nit
pgayvallet Nov 11, 2024
3a08718
[CI] Auto-commit changed files from 'node scripts/build_plugin_list_d…
kibanamachine Nov 11, 2024
4feb090
update mappings
pgayvallet Nov 11, 2024
7bb2322
[CI] Auto-commit changed files from 'node scripts/jest_integration -u…
kibanamachine Nov 11, 2024
7af9f4e
add more unit tests for retrieveDocumentation
pgayvallet Nov 11, 2024
c00c642
use task manager for installation workflows
pgayvallet Nov 12, 2024
5fdd08d
[CI] Auto-commit changed files from 'node scripts/notice'
kibanamachine Nov 12, 2024
0764c0e
create global manager for documentation installation
pgayvallet Nov 13, 2024
be74940
fix getTaskStatus
pgayvallet Nov 13, 2024
3baf56d
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Nov 13, 2024
5f7dbda
fixes due to merge
pgayvallet Nov 13, 2024
27abf46
add audit logging for product doc
pgayvallet Nov 13, 2024
34ba91c
add tests for doc manager
pgayvallet Nov 14, 2024
407430d
fix response ops ftr test
pgayvallet Nov 14, 2024
f2cd9e6
remove semantic text for ai_subtitle
pgayvallet Nov 14, 2024
3858b63
tweak summarize prompt
pgayvallet Nov 14, 2024
75da6ea
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Nov 14, 2024
1ddad2f
[CI] Auto-commit changed files from 'make api-docs'
kibanamachine Nov 14, 2024
7063ce4
fix type in test
pgayvallet Nov 15, 2024
c5c9d76
Merge remote-tracking branch 'upstream/main' into kbn-193849-add-kb-r…
pgayvallet Nov 18, 2024
a34e114
update doc
pgayvallet Nov 18, 2024
b9145ef
check status after installation
pgayvallet Nov 18, 2024
040956d
remove unused return type
pgayvallet Nov 18, 2024
09f8b2d
Merge branch 'main' into kbn-193849-add-kb-registry-plugin
elasticmachine Nov 19, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -768,6 +768,7 @@ x-pack/examples/triggers_actions_ui_example @elastic/response-ops
x-pack/examples/ui_actions_enhanced_examples @elastic/appex-sharedux
x-pack/packages/ai-infra/inference-common @elastic/appex-ai-infra
x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra
x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra
x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared @elastic/kibana-management
x-pack/packages/index-management/index_management_shared_types @elastic/kibana-management
x-pack/packages/kbn-ai-assistant @elastic/search-kibana
Expand Down Expand Up @@ -857,6 +858,8 @@ x-pack/packages/security/role_management_model @elastic/kibana-security
x-pack/packages/security/ui_components @elastic/kibana-security
x-pack/performance @elastic/appex-qa
x-pack/plugins/actions @elastic/response-ops
x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra
x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra
x-pack/plugins/aiops @elastic/ml-ui
x-pack/plugins/alerting @elastic/response-ops
x-pack/plugins/banners @elastic/appex-sharedux
Expand Down
8 changes: 8 additions & 0 deletions docs/developer/plugin-list.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana.
using the CURL scripts in the scripts folder.


|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks]
|This plugin contains various LLM tasks.


|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess]
|Exposes services to access logs data.

Expand Down Expand Up @@ -767,6 +771,10 @@ Elastic.
|This plugin helps users learn how to use the Painless scripting language.


|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase]
|This plugin contains the product documentation base service.


|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling]
|Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views.

Expand Down
9 changes: 9 additions & 0 deletions docs/user/security/audit-logging.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors.
| `success` | Creating trained model.
| `failure` | Failed to create trained model.

.1+| `product_documentation_create`
| `unknown` | User requested to install the product documentation for use in AI Assistants.

3+a|
====== Type: change

Expand Down Expand Up @@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors.
| `success` | Updating trained model deployment.
| `failure` | Failed to update trained model deployment.

.1+| `product_documentation_update`
| `unknown` | User requested to update the product documentation for use in AI Assistants.

3+a|
====== Type: deletion

Expand Down Expand Up @@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors.
| `success` | Deleting trained model.
| `failure` | Failed to delete trained model.

.1+| `product_documentation_delete`
| `unknown` | User requested to delete the product documentation for use in AI Assistants.

3+a|
====== Type: access

Expand Down
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,7 @@
"@kbn/licensing-plugin": "link:x-pack/plugins/licensing",
"@kbn/links-plugin": "link:src/plugins/links",
"@kbn/lists-plugin": "link:x-pack/plugins/lists",
"@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks",
"@kbn/locator-examples-plugin": "link:examples/locator_examples",
"@kbn/locator-explorer-plugin": "link:examples/locator_explorer",
"@kbn/logging": "link:packages/kbn-logging",
Expand Down Expand Up @@ -721,6 +722,8 @@
"@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel",
"@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing",
"@kbn/presentation-util-plugin": "link:src/plugins/presentation_util",
"@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base",
"@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common",
"@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access",
"@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling",
"@kbn/profiling-utils": "link:packages/kbn-profiling-utils",
Expand Down
7 changes: 7 additions & 0 deletions packages/kbn-check-mappings-update-cli/current_fields.json
Original file line number Diff line number Diff line change
Expand Up @@ -855,6 +855,13 @@
"policy-settings-protection-updates-note": [
"note"
],
"product-doc-install-status": [
"index_name",
"installation_status",
"last_installation_date",
"product_name",
"product_version"
],
"query": [
"description",
"title",
Expand Down
20 changes: 20 additions & 0 deletions packages/kbn-check-mappings-update-cli/current_mappings.json
Original file line number Diff line number Diff line change
Expand Up @@ -2841,6 +2841,26 @@
}
}
},
"product-doc-install-status": {
"dynamic": false,
"properties": {
"index_name": {
"type": "keyword"
},
"installation_status": {
"type": "keyword"
},
"last_installation_date": {
"type": "date"
},
"product_name": {
"type": "keyword"
},
"product_version": {
"type": "keyword"
}
}
},
"query": {
"dynamic": false,
"properties": {
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-optimizer/limits.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ pageLoadAssetSize:
painlessLab: 179748
presentationPanel: 55463
presentationUtil: 58834
productDocBase: 22500
profiling: 36694
remoteClusters: 51327
reporting: 58600
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75",
"osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4",
"policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352",
"product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c",
"query": "501bece68f26fe561286a488eabb1a8ab12f1137",
"risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb",
"rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ const previouslyRegisteredTypes = [
'osquery-usage-metric',
'osquery-manager-usage-metric',
'policy-settings-protection-updates-note',
'product-doc-install-status',
'query',
'rules-settings',
'sample-data-telemetry',
Expand Down
6 changes: 6 additions & 0 deletions tsconfig.base.json
Original file line number Diff line number Diff line change
Expand Up @@ -1148,6 +1148,8 @@
"@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"],
"@kbn/lists-plugin": ["x-pack/plugins/lists"],
"@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"],
"@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"],
"@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"],
"@kbn/locator-examples-plugin": ["examples/locator_examples"],
"@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"],
"@kbn/locator-explorer-plugin": ["examples/locator_explorer"],
Expand Down Expand Up @@ -1390,6 +1392,10 @@
"@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"],
"@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"],
"@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"],
"@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"],
"@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"],
"@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"],
"@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"],
"@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"],
"@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"],
"@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,49 @@
# @kbn/product-doc-artifact-builder

Script to build the knowledge base artifacts
Script to build the knowledge base artifacts.

## How to run

```
node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product}
```

### parameters

#### `stack-version`:

the stack version to generate the artifacts for.

#### `product-name`:

(multi-value) the list of products to generate artifacts for.

possible values:
- "kibana"
- "elasticsearch"
- "observability"
- "security"

#### `target-folder`:

The folder to generate the artifacts in.

Defaults to `{REPO_ROOT}/build-kb-artifacts`.

#### `build-folder`:

The folder to use for temporary files.

Defaults to `{REPO_ROOT}/build/temp-kb-artifacts`

#### Cluster infos

- params for the source cluster:
`sourceClusterUrl` / env.KIBANA_SOURCE_CLUSTER_URL
`sourceClusterUsername` / env.KIBANA_SOURCE_CLUSTER_USERNAME
`sourceClusterPassword` / env.KIBANA_SOURCE_CLUSTER_PASSWORD

- params for the embedding cluster:
`embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL
`embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME
`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,13 @@
* 2.0.
*/

export interface ArtifactManifest {
formatVersion: string;
productName: string;
productVersion: string;
}
import type { ArtifactManifest, ProductName } from '@kbn/product-doc-common';

export const getArtifactManifest = ({
productName,
stackVersion,
}: {
productName: string;
productName: ProductName;
stackVersion: string;
}): ArtifactManifest => {
return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@ export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMappi
slug: { type: 'keyword' },
url: { type: 'keyword' },
version: { type: 'version' },
ai_subtitle: {
type: 'semantic_text',
inference_id: inferenceEndpoint,
},
ai_subtitle: { type: 'text' },
ai_summary: {
type: 'semantic_text',
inference_id: inferenceEndpoint,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,34 @@
* 2.0.
*/

/**
* The allowed product names, as found in the source's cluster
*/
export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability'];
import type { ProductName } from '@kbn/product-doc-common';

const productNameToSourceNamesMap: Record<ProductName, string[]> = {
kibana: ['Kibana'],
elasticsearch: ['Elasticsearch'],
security: ['Security'],
observability: ['Observability'],
};

const sourceNameToProductName = Object.entries(productNameToSourceNamesMap).reduce<
Record<string, ProductName>
>((map, [productName, sourceNames]) => {
sourceNames.forEach((sourceName) => {
map[sourceName] = productName as ProductName;
});
return map;
}, {});

export const getSourceNamesFromProductName = (productName: ProductName): string[] => {
if (!productNameToSourceNamesMap[productName]) {
throw new Error(`Unknown product name: ${productName}`);
}
return productNameToSourceNamesMap[productName];
};

export const getProductNameFromSource = (source: string): ProductName => {
if (!sourceNameToProductName[source]) {
throw new Error(`Unknown source name: ${source}`);
}
return sourceNameToProductName[source];
};
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import Path from 'path';
import { Client } from '@elastic/elasticsearch';
import { ToolingLog } from '@kbn/tooling-log';
import type { ProductName } from '@kbn/product-doc-common';
import {
// checkConnectivity,
createTargetIndex,
Expand All @@ -18,6 +19,7 @@ import {
createArtifact,
cleanupFolders,
deleteIndex,
processDocuments,
} from './tasks';
import type { TaskConfig } from './types';

Expand Down Expand Up @@ -93,7 +95,7 @@ const buildArtifact = async ({
sourceClient,
log,
}: {
productName: string;
productName: ProductName;
stackVersion: string;
buildFolder: string;
targetFolder: string;
Expand All @@ -105,14 +107,16 @@ const buildArtifact = async ({

const targetIndex = getTargetIndexName({ productName, stackVersion });

const documents = await extractDocumentation({
let documents = await extractDocumentation({
client: sourceClient,
index: 'search-docs-1',
log,
productName,
stackVersion,
});

documents = await processDocuments({ documents, log });

await createTargetIndex({
client: embeddingClient,
indexName: targetIndex,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,19 @@
*/

import Path from 'path';
import { REPO_ROOT } from '@kbn/repo-info';
import yargs from 'yargs';
import { REPO_ROOT } from '@kbn/repo-info';
import { DocumentationProduct } from '@kbn/product-doc-common';
import type { TaskConfig } from './types';
import { buildArtifacts } from './build_artifacts';
import { sourceProductNames } from './artifact/product_name';

function options(y: yargs.Argv) {
return y
.option('productName', {
describe: 'name of products to generate documentation for',
array: true,
choices: sourceProductNames,
default: ['Kibana'],
choices: Object.values(DocumentationProduct),
default: [DocumentationProduct.kibana],
})
.option('stackVersion', {
describe: 'The stack version to generate documentation for',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import Path from 'path';
import AdmZip from 'adm-zip';
import type { ToolingLog } from '@kbn/tooling-log';
import { getArtifactName, type ProductName } from '@kbn/product-doc-common';
import { getArtifactMappings } from '../artifact/mappings';
import { getArtifactManifest } from '../artifact/manifest';
import { getArtifactName } from '../artifact/artifact_name';

export const createArtifact = async ({
productName,
Expand All @@ -21,7 +21,7 @@ export const createArtifact = async ({
}: {
buildFolder: string;
targetFolder: string;
productName: string;
productName: ProductName;
stackVersion: string;
log: ToolingLog;
}) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import Fs from 'fs/promises';
import type { Client } from '@elastic/elasticsearch';
import type { ToolingLog } from '@kbn/tooling-log';

const fileSizeLimit = 250_000;
const fileSizeLimit = 500_000;

export const createChunkFiles = async ({
index,
Expand Down
Loading