diff --git a/.buildkite/scripts/steps/functional/performance_playwright.sh b/.buildkite/scripts/steps/functional/performance_playwright.sh index d84bf77ed54cf..2787d6f5b68a0 100644 --- a/.buildkite/scripts/steps/functional/performance_playwright.sh +++ b/.buildkite/scripts/steps/functional/performance_playwright.sh @@ -39,8 +39,13 @@ if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-performance-data-set-extraction" ]; t node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --skip-warmup else # pipeline should use bare metal static worker - echo "--- Running performance tests" - node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" + if [[ -z "${JOURNEYS_GROUP+x}" ]]; then + echo "--- Running performance tests" + node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" + else + echo "--- Running performance tests: '$JOURNEYS_GROUP' group" + node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --group "$JOURNEYS_GROUP" + fi fi echo "--- Upload journey step screenshots" diff --git a/.devcontainer/.env.template b/.devcontainer/.env.template index 3ca02c49bfa9c..b812dc9659a02 100644 --- a/.devcontainer/.env.template +++ b/.devcontainer/.env.template @@ -1,4 +1,5 @@ # /bin/bash or /bin/zsh (oh-my-zsh is installed by default as well) SHELL=/bin/bash -# Switch to 1 to enable FIPS environment, any other value to disable +# Switch to 1 to enable FIPS environment, any other value to disable, +# then close and reopen a new terminal to setup the environment FIPS=0 diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 539e23a4a3a31..142e5ce933777 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -49,6 +49,9 @@ WORKDIR ${KBN_DIR} # Node and NVM setup COPY .node-version /tmp/ + +USER vscode + RUN mkdir -p $NVM_DIR && \ curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh | bash && \ . "$NVM_DIR/nvm.sh" && \ @@ -61,6 +64,8 @@ RUN mkdir -p $NVM_DIR && \ echo "source $NVM_DIR/nvm.sh" >> ${HOME}/.zshrc && \ chown -R 1000:1000 "${HOME}/.npm" +USER root + # Reload the env everytime a new shell is opened incase the .env file changed. RUN echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.bashrc && \ echo "source $KBN_DIR/.devcontainer/scripts/env.sh" >> ${HOME}/.zshrc diff --git a/dev_docs/tutorials/performance/adding_performance_journey.mdx b/dev_docs/tutorials/performance/adding_performance_journey.mdx index 2f7f37452c285..a9c4fe7ef6bc5 100644 --- a/dev_docs/tutorials/performance/adding_performance_journey.mdx +++ b/dev_docs/tutorials/performance/adding_performance_journey.mdx @@ -89,6 +89,27 @@ simulate real life internet connection. This means that all requests have a fixe In order to keep track on performance metrics stability, journeys are run on main branch with a scheduled interval. Bare metal machine is used to produce results as stable and reproducible as possible. +#### Running subset of journeys for the PR + +Some code changes might affect the Kibana performance and it might be benefitial to run relevant journeys against the PR +and compare performance metrics vs. the ones on main branch. + +In oder to trigger the build for Kibana PR, you can follow these steps: + +- Create a new kibana-single-user-performance [build](https://buildkite.com/elastic/kibana-single-user-performance#new) +- Provide the following arguments: + - Branch: `refs/pull//head` + - Under Options, set the environment variable: `JOURNEYS_GROUP=` + +Currently supported journey groups: + +- kibanaStartAndLoad +- crud +- dashboard +- discover +- maps +- ml + #### Machine specifications All benchmarks are run on bare-metal machines with the [following specifications](https://www.hetzner.com/dedicated-rootserver/ex100): diff --git a/package.json b/package.json index a152b84fb6592..49b45c263c020 100644 --- a/package.json +++ b/package.json @@ -1801,7 +1801,7 @@ "tape": "^5.0.1", "terser": "^5.32.0", "terser-webpack-plugin": "^4.2.3", - "tough-cookie": "^4.1.4", + "tough-cookie": "^5.0.0", "tree-kill": "^1.2.2", "ts-morph": "^15.1.0", "tsd": "^0.31.1", diff --git a/packages/kbn-esql-ast/src/antlr_error_listener.ts b/packages/kbn-esql-ast/src/antlr_error_listener.ts index e685dfe3473ba..5bd02b3b74eb9 100644 --- a/packages/kbn-esql-ast/src/antlr_error_listener.ts +++ b/packages/kbn-esql-ast/src/antlr_error_listener.ts @@ -12,7 +12,7 @@ import { ErrorListener } from 'antlr4'; import type { EditorError } from './types'; import { getPosition } from './ast_position_utils'; -const REPLACE_DEV = /,*\s*DEV_\w+\s*/g; +const REPLACE_DEV = /,{0,1}(? { protected errors: EditorError[] = []; diff --git a/packages/kbn-monaco/src/console/lexer_rules/console_output.ts b/packages/kbn-monaco/src/console/lexer_rules/console_output.ts index c9ebd0cb6e876..30b846d529fe5 100644 --- a/packages/kbn-monaco/src/console/lexer_rules/console_output.ts +++ b/packages/kbn-monaco/src/console/lexer_rules/console_output.ts @@ -25,21 +25,24 @@ export const consoleOutputLexerRules: monaco.languages.IMonarchLanguage = { comments: [ // Line comment indicated by # // Everything after the # character is matched, stopping right before the status code and status text at the end if they are present - matchTokensWithEOL('comment', /# .+?(?=\s+\d{3}(?: \w+)*$)/, 'root', 'status'), + matchTokensWithEOL('comment.default', /# .+?(?=\s+\[\b1\d{2}(?: \w+)*\]$)/, 'root', 'status'), + matchTokensWithEOL('comment.success', /# .+?(?=\s+\[\b2\d{2}(?: \w+)*\]$)/, 'root', 'status'), + matchTokensWithEOL('comment.primary', /# .+?(?=\s+\[\b3\d{2}(?: \w+)*\]$)/, 'root', 'status'), + matchTokensWithEOL('comment.warning', /# .+?(?=\s+\[\b4\d{2}(?: \w+)*\]$)/, 'root', 'status'), + matchTokensWithEOL('comment.danger', /# .+?(?=\s+\[\b5\d{2}(?: \w+)*\]$)/, 'root', 'status'), ...consoleSharedLexerRules.tokenizer.comments, ], status: [ - // Following HTTP response status codes conventions - // Informational responses (status codes 100 – 199) - matchTokensWithEOL('status.info', /\b1\d{2}(?: \w+)*$/, 'root'), - // Successful responses (status codes 200 – 299) - matchTokensWithEOL('status.success', /\b2\d{2}(?: \w+)*$/, 'root'), - // Redirection messages (status codes 300 – 399) - matchTokensWithEOL('status.redirect', /\b3\d{2}(?: \w+)*$/, 'root'), - // Client error responses (status codes 400 – 499) - matchTokensWithEOL('status.warning', /\b4\d{2}(?: \w+)*$/, 'root'), - // Server error responses (status codes 500 – 599) - matchTokensWithEOL('status.error', /\b5\d{2}(?: \w+)*$/, 'root'), + // Status codes 100 – 199 + matchTokensWithEOL('status.default', /\[\b1\d{2}(?: \w+)*\]$/, 'root'), + // Status codes 200 – 299 + matchTokensWithEOL('status.success', /\[\b2\d{2}(?: \w+)*\]$/, 'root'), + // Status codes 300 – 399 + matchTokensWithEOL('status.primary', /\[\b3\d{2}(?: \w+)*\]$/, 'root'), + // Status codes 400 – 499 + matchTokensWithEOL('status.warning', /\[\b4\d{2}(?: \w+)*\]$/, 'root'), + // Status codes 500 – 599 + matchTokensWithEOL('status.danger', /\[\b5\d{2}(?: \w+)*\]$/, 'root'), ], }, }; diff --git a/packages/kbn-monaco/src/console/theme.ts b/packages/kbn-monaco/src/console/theme.ts index bcb3d93368018..6f5a1e78618fb 100644 --- a/packages/kbn-monaco/src/console/theme.ts +++ b/packages/kbn-monaco/src/console/theme.ts @@ -20,6 +20,11 @@ const background = euiThemeVars.euiFormBackgroundColor; const booleanTextColor = '#585CF6'; const methodTextColor = '#DD0A73'; const urlTextColor = '#00A69B'; +const defaultStatusBackgroundColor = darkMode ? '#191B20' : '#F7F8FA'; +const successStatusBackgroundColor = darkMode ? '#212B30' : '#E7F5F5'; +const primaryStatusBackgroundColor = darkMode ? '#1E232D' : '#EBF1F7'; +const warningStatusBackgroundColor = darkMode ? '#2C2B25' : '#FBF6E9'; +const dangerStatusBackgroundColor = darkMode ? '#2E2024' : '#F6E6E7'; export const buildConsoleTheme = (): monaco.editor.IStandaloneThemeData => { const euiTheme = darkMode ? buildDarkTheme() : buildLightTheme(); return { @@ -39,27 +44,56 @@ export const buildConsoleTheme = (): monaco.editor.IStandaloneThemeData => { makeHighContrastColor(euiThemeVars.euiColorAccentText)(background) ), ...buildRuleGroup( - ['status.info'], - makeHighContrastColor(euiThemeVars.euiTextColor)(background) + ['comment.default'], + makeHighContrastColor(euiThemeVars.euiTextColor)(defaultStatusBackgroundColor) + ), + ...buildRuleGroup( + ['comment.success'], + makeHighContrastColor(euiThemeVars.euiColorSuccessText)(successStatusBackgroundColor) + ), + ...buildRuleGroup( + ['comment.primary'], + makeHighContrastColor(euiThemeVars.euiTextColor)(primaryStatusBackgroundColor) + ), + ...buildRuleGroup( + ['comment.warning'], + makeHighContrastColor(euiThemeVars.euiColorWarningText)(warningStatusBackgroundColor) + ), + ...buildRuleGroup( + ['comment.danger'], + makeHighContrastColor(euiThemeVars.euiColorDangerText)(dangerStatusBackgroundColor) + ), + ...buildRuleGroup( + ['status.default'], + makeHighContrastColor(euiThemeVars.euiTextColor)(defaultStatusBackgroundColor), + true ), ...buildRuleGroup( ['status.success'], - makeHighContrastColor(euiThemeVars.euiTextColor)(euiThemeVars.euiColorSuccess) + makeHighContrastColor(euiThemeVars.euiColorSuccessText)(successStatusBackgroundColor), + true ), ...buildRuleGroup( - ['status.redirect'], - makeHighContrastColor(euiThemeVars.euiTextColor)(background) + ['status.primary'], + makeHighContrastColor(euiThemeVars.euiTextColor)(primaryStatusBackgroundColor), + true ), ...buildRuleGroup( ['status.warning'], - makeHighContrastColor(euiThemeVars.euiTextColor)(euiThemeVars.euiColorWarning) + makeHighContrastColor(euiThemeVars.euiColorWarningText)(warningStatusBackgroundColor), + true ), ...buildRuleGroup( - ['status.error'], - makeHighContrastColor('#FFFFFF')(euiThemeVars.euiColorDanger) + ['status.danger'], + makeHighContrastColor(euiThemeVars.euiColorDangerText)(dangerStatusBackgroundColor), + true ), ...buildRuleGroup(['method'], makeHighContrastColor(methodTextColor)(background)), ...buildRuleGroup(['url'], makeHighContrastColor(urlTextColor)(background)), ], + colors: { + ...euiTheme.colors, + 'editorLineNumber.foreground': euiThemeVars.euiTextColor, + }, }; }; diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index fc9120e99bde9..a7d94a423d606 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -119,7 +119,7 @@ pageLoadAssetSize: observabilityAiAssistantManagement: 19279 observabilityLogsExplorer: 46650 observabilityOnboarding: 19573 - observabilityShared: 72039 + observabilityShared: 80000 osquery: 107090 painlessLab: 179748 presentationPanel: 55463 diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list/create_endpoint_list.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list/create_endpoint_list.schema.yaml index 9703e31287ae6..b2ca0f563617d 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list/create_endpoint_list.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list/create_endpoint_list.schema.yaml @@ -8,8 +8,8 @@ paths: x-labels: [serverless, ess] x-codegen-enabled: true operationId: CreateEndpointList - summary: Creates an endpoint list - description: Creates an endpoint list or does nothing if the list already exists + summary: Create an endpoint exception list + description: Create an endpoint exception list, which groups endpoint exception list items. If an endpoint exception list already exists, an empty response is returned. responses: 200: description: Successful response diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list_item/create_endpoint_list_item.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list_item/create_endpoint_list_item.schema.yaml index 8807f6cf76f7c..b90bee75fc073 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list_item/create_endpoint_list_item.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/api/create_endpoint_list_item/create_endpoint_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] x-codegen-enabled: true operationId: CreateEndpointListItem - summary: Creates an endpoint list item + summary: Create an endpoint exception list item + description: Create an endpoint exception list item, and associate it with the endpoint exception list. requestBody: description: Exception list item's properties required: true diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/api/delete_endpoint_list_item/delete_endpoint_list_item.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/api/delete_endpoint_list_item/delete_endpoint_list_item.schema.yaml index cd0ddd9dd69cb..69db506169187 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/api/delete_endpoint_list_item/delete_endpoint_list_item.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/api/delete_endpoint_list_item/delete_endpoint_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] x-codegen-enabled: true operationId: DeleteEndpointListItem - summary: Deletes an endpoint list item + summary: Delete an endpoint exception list item + description: Delete an endpoint exception list item using the `id` or `item_id` field. parameters: - name: id in: query diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/api/find_endpoint_list_item/find_endpoint_list_item.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/api/find_endpoint_list_item/find_endpoint_list_item.schema.yaml index e727367ff9c7d..6dc2fcaa7e87a 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/api/find_endpoint_list_item/find_endpoint_list_item.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/api/find_endpoint_list_item/find_endpoint_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] x-codegen-enabled: true operationId: FindEndpointListItems - summary: Finds endpoint list items + summary: Get endpoint exception list items + description: Get a list of all endpoint exception list items. parameters: - name: filter in: query diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/api/read_endpoint_list_item/read_endpoint_list_item.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/api/read_endpoint_list_item/read_endpoint_list_item.schema.yaml index 81be7c67a00b3..8a4e0b291c76f 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/api/read_endpoint_list_item/read_endpoint_list_item.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/api/read_endpoint_list_item/read_endpoint_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] x-codegen-enabled: true operationId: ReadEndpointListItem - summary: Reads an endpoint list item + summary: Get an endpoint exception list item + description: Get the details of an endpoint exception list item using the `id` or `item_id` field. parameters: - name: id in: query diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/api/update_endpoint_list_item/update_endpoint_list_item.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/api/update_endpoint_list_item/update_endpoint_list_item.schema.yaml index dcbf24be28763..679000674c7b2 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/api/update_endpoint_list_item/update_endpoint_list_item.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/api/update_endpoint_list_item/update_endpoint_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] x-codegen-enabled: true operationId: UpdateEndpointListItem - summary: Updates an endpoint list item + summary: Update an endpoint exception list item + description: Update an endpoint exception list item using the `id` or `item_id` field. requestBody: description: Exception list item's properties required: true diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/ess/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/ess/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml index cbd98091ca378..d3f3ca4d2c846 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/ess/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/ess/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml @@ -13,7 +13,10 @@ servers: paths: /api/endpoint_list: post: - description: Creates an endpoint list or does nothing if the list already exists + description: >- + Create an endpoint exception list, which groups endpoint exception list + items. If an endpoint exception list already exists, an empty response + is returned. operationId: CreateEndpointList responses: '200': @@ -48,9 +51,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Creates an endpoint list + summary: Create an endpoint exception list /api/endpoint_list/items: delete: + description: >- + Delete an endpoint exception list item using the `id` or `item_id` + field. operationId: DeleteEndpointListItem parameters: - description: Either `id` or `item_id` must be specified @@ -104,8 +110,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Deletes an endpoint list item + summary: Delete an endpoint exception list item get: + description: >- + Get the details of an endpoint exception list item using the `id` or + `item_id` field. operationId: ReadEndpointListItem parameters: - description: Either `id` or `item_id` must be specified @@ -161,8 +170,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Reads an endpoint list item + summary: Get an endpoint exception list item post: + description: >- + Create an endpoint exception list item, and associate it with the + endpoint exception list. operationId: CreateEndpointListItem requestBody: content: @@ -237,8 +249,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Creates an endpoint list item + summary: Create an endpoint exception list item put: + description: >- + Update an endpoint exception list item using the `id` or `item_id` + field. operationId: UpdateEndpointListItem requestBody: content: @@ -318,9 +333,10 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Updates an endpoint list item + summary: Update an endpoint exception list item /api/endpoint_list/items/_find: get: + description: Get a list of all endpoint exception list items. operationId: FindEndpointListItems parameters: - description: > @@ -422,7 +438,7 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Finds endpoint list items + summary: Get endpoint exception list items components: schemas: EndpointList: diff --git a/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/serverless/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml b/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/serverless/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml index e8a8966c18586..4222cb062ba24 100644 --- a/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/serverless/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml +++ b/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/serverless/security_solution_endpoint_exceptions_api_2023_10_31.bundled.schema.yaml @@ -13,7 +13,10 @@ servers: paths: /api/endpoint_list: post: - description: Creates an endpoint list or does nothing if the list already exists + description: >- + Create an endpoint exception list, which groups endpoint exception list + items. If an endpoint exception list already exists, an empty response + is returned. operationId: CreateEndpointList responses: '200': @@ -48,9 +51,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Creates an endpoint list + summary: Create an endpoint exception list /api/endpoint_list/items: delete: + description: >- + Delete an endpoint exception list item using the `id` or `item_id` + field. operationId: DeleteEndpointListItem parameters: - description: Either `id` or `item_id` must be specified @@ -104,8 +110,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Deletes an endpoint list item + summary: Delete an endpoint exception list item get: + description: >- + Get the details of an endpoint exception list item using the `id` or + `item_id` field. operationId: ReadEndpointListItem parameters: - description: Either `id` or `item_id` must be specified @@ -161,8 +170,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Reads an endpoint list item + summary: Get an endpoint exception list item post: + description: >- + Create an endpoint exception list item, and associate it with the + endpoint exception list. operationId: CreateEndpointListItem requestBody: content: @@ -237,8 +249,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Creates an endpoint list item + summary: Create an endpoint exception list item put: + description: >- + Update an endpoint exception list item using the `id` or `item_id` + field. operationId: UpdateEndpointListItem requestBody: content: @@ -318,9 +333,10 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Updates an endpoint list item + summary: Update an endpoint exception list item /api/endpoint_list/items/_find: get: + description: Get a list of all endpoint exception list items. operationId: FindEndpointListItems parameters: - description: > @@ -422,7 +438,7 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error - summary: Finds endpoint list items + summary: Get endpoint exception list items components: schemas: EndpointList: diff --git a/packages/kbn-text-based-editor/src/helpers.ts b/packages/kbn-text-based-editor/src/helpers.ts index fb541c4fe39be..0ba6d5004606f 100644 --- a/packages/kbn-text-based-editor/src/helpers.ts +++ b/packages/kbn-text-based-editor/src/helpers.ts @@ -77,12 +77,14 @@ export const parseWarning = (warning: string): MonacoMessage[] => { startColumn = Number(encodedColumn); startLineNumber = Number(encodedLine.replace('Line ', '')); } - // extract the length of the "expression" within the message - // and try to guess the correct size for the editor marker to highlight - if (/\[.*\]/.test(warningMessage)) { - const [_, wordWithError] = warningMessage.split('['); - if (wordWithError) { - errorLength = wordWithError.length; + const openingSquareBracketIndex = warningMessage.indexOf('['); + if (openingSquareBracketIndex !== -1) { + const closingSquareBracketIndex = warningMessage.indexOf( + ']', + openingSquareBracketIndex + ); + if (closingSquareBracketIndex !== -1) { + errorLength = warningMessage.length - openingSquareBracketIndex - 1; } } } diff --git a/packages/kbn-typed-react-router-config/src/types/index.ts b/packages/kbn-typed-react-router-config/src/types/index.ts index 8f2daba368fa3..3b4c36c42af53 100644 --- a/packages/kbn-typed-react-router-config/src/types/index.ts +++ b/packages/kbn-typed-react-router-config/src/types/index.ts @@ -191,204 +191,3 @@ type MapRoutes extends Record ? FromRouteMap : never; - -// const element = null as any; - -// const routes = { -// '/link-to/transaction/{transactionId}': { -// element, -// }, -// '/link-to/trace/{traceId}': { -// element, -// }, -// '/': { -// element, -// children: { -// '/settings': { -// element, -// children: { -// '/settings/agent-configuration': { -// element, -// }, -// '/settings/agent-configuration/create': { -// element, -// params: t.partial({ -// query: t.partial({ -// pageStep: t.string, -// }), -// }), -// }, -// '/settings/agent-configuration/edit': { -// element, -// params: t.partial({ -// query: t.partial({ -// pageStep: t.string, -// }), -// }), -// }, -// '/settings/apm-indices': { -// element, -// }, -// '/settings/custom-links': { -// element, -// }, -// '/settings/schema': { -// element, -// }, -// '/settings/anomaly-detection': { -// element, -// }, -// '/settings/agent-keys': { -// element, -// }, -// '/settings': { -// element, -// }, -// }, -// }, -// '/services/:serviceName': { -// element, -// params: t.intersection([ -// t.type({ -// path: t.type({ -// serviceName: t.string, -// }), -// }), -// t.partial({ -// query: t.partial({ -// environment: t.string, -// rangeFrom: t.string, -// rangeTo: t.string, -// comparisonEnabled: t.string, -// comparisonType: t.string, -// latencyAggregationType: t.string, -// transactionType: t.string, -// kuery: t.string, -// }), -// }), -// ]), -// children: { -// '/services/:serviceName/overview': { -// element, -// }, -// '/services/:serviceName/transactions': { -// element, -// }, -// '/services/:serviceName/transactions/view': { -// element, -// }, -// '/services/:serviceName/dependencies': { -// element, -// }, -// '/services/:serviceName/errors': { -// element, -// children: { -// '/services/:serviceName/errors/:groupId': { -// element, -// params: t.type({ -// path: t.type({ -// groupId: t.string, -// }), -// }), -// }, -// '/services/:serviceName/errors': { -// element, -// params: t.partial({ -// query: t.partial({ -// sortDirection: t.string, -// sortField: t.string, -// pageSize: t.string, -// page: t.string, -// }), -// }), -// }, -// }, -// }, -// '/services/:serviceName/metrics': { -// element, -// }, -// '/services/:serviceName/nodes': { -// element, -// children: { -// '/services/{serviceName}/nodes/{serviceNodeName}/metrics': { -// element, -// }, -// '/services/:serviceName/nodes': { -// element, -// }, -// }, -// }, -// '/services/:serviceName/service-map': { -// element, -// }, -// '/services/:serviceName/logs': { -// element, -// }, -// '/services/:serviceName/profiling': { -// element, -// }, -// '/services/:serviceName': { -// element, -// }, -// }, -// }, -// '/': { -// element, -// params: t.partial({ -// query: t.partial({ -// rangeFrom: t.string, -// rangeTo: t.string, -// }), -// }), -// children: { -// '/services': { -// element, -// }, -// '/traces': { -// element, -// }, -// '/service-map': { -// element, -// }, -// '/backends': { -// element, -// children: { -// '/backends/{backendName}/overview': { -// element, -// }, -// '/backends/overview': { -// element, -// }, -// '/backends': { -// element, -// }, -// }, -// }, -// '/': { -// element, -// }, -// }, -// }, -// }, -// }, -// }; - -// type Routes = typeof routes; - -// type Mapped = MapRoutes; -// type Paths = PathsOf; - -// type Bar = Match; -// type Foo = OutputOf; -// type Baz = OutputOf; - -// const { path }: Foo = {} as any; - -// function _useApmParams>(p: TPath): OutputOf { -// return {} as any; -// } - -// const { -// path: { serviceName }, -// query: { comparisonType }, -// } = _useApmParams('/services/:serviceName/nodes/*'); diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx index c6f301bbdf69f..658bf96dc76c9 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx @@ -14,6 +14,7 @@ import { UiCounterMetricType } from '@kbn/analytics'; import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public'; import { Draggable } from '@kbn/dom-drag-drop'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; +import { Filter } from '@kbn/es-query'; import type { SearchMode } from '../../types'; import { FieldItemButton, type FieldItemButtonProps } from '../../components/field_item_button'; import { @@ -200,6 +201,10 @@ export interface UnifiedFieldListItemProps { * Item size */ size: FieldItemButtonProps['size']; + /** + * Custom filters to apply for the field list, ex: namespace custom filter + */ + additionalFilters?: Filter[]; } function UnifiedFieldListItemComponent({ @@ -223,6 +228,7 @@ function UnifiedFieldListItemComponent({ groupIndex, itemIndex, size, + additionalFilters, }: UnifiedFieldListItemProps) { const [infoIsOpen, setOpen] = useState(false); @@ -288,6 +294,7 @@ function UnifiedFieldListItemComponent({ multiFields={multiFields} dataView={dataView} onAddFilter={addFilterAndClosePopover} + additionalFilters={additionalFilters} /> {searchMode === 'documents' && multiFields && ( diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item_stats.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item_stats.tsx index c83a0694c7b67..223a5e15ca6e7 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item_stats.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item_stats.tsx @@ -27,10 +27,11 @@ export interface UnifiedFieldListItemStatsProps { dataView: DataView; multiFields?: Array<{ field: DataViewField; isSelected: boolean }>; onAddFilter: FieldStatsProps['onAddFilter']; + additionalFilters?: FieldStatsProps['filters']; } export const UnifiedFieldListItemStats: React.FC = React.memo( - ({ stateService, services, field, dataView, multiFields, onAddFilter }) => { + ({ stateService, services, field, dataView, multiFields, onAddFilter, additionalFilters }) => { const querySubscriberResult = useQuerySubscriber({ data: services.data, timeRangeUpdatesType: stateService.creationOptions.timeRangeUpdatesType, @@ -55,6 +56,11 @@ export const UnifiedFieldListItemStats: React.FC [services] ); + const filters = useMemo( + () => [...(querySubscriberResult.filters ?? []), ...(additionalFilters ?? [])], + [querySubscriberResult.filters, additionalFilters] + ); + if (!hasQuerySubscriberData(querySubscriberResult)) { return null; } @@ -63,7 +69,7 @@ export const UnifiedFieldListItemStats: React.FC & { /** * All fields: fields from data view and unmapped fields or columns from text-based search @@ -168,6 +169,7 @@ export const UnifiedFieldListSidebarComponent: React.FC { const { dataViews, core } = services; const useNewFieldsApi = useMemo( @@ -285,6 +287,7 @@ export const UnifiedFieldListSidebarComponent: React.FC ), @@ -304,6 +307,7 @@ export const UnifiedFieldListSidebarComponent: React.FC( createStateService({ options: getCreationOptions() }) @@ -151,11 +152,16 @@ const UnifiedFieldListSidebarContainer = memo( const searchMode: SearchMode | undefined = querySubscriberResult.searchMode; const isAffectedByGlobalFilter = Boolean(querySubscriberResult.filters?.length); + const filters = useMemo( + () => [...(querySubscriberResult.filters ?? []), ...(additionalFilters ?? [])], + [querySubscriberResult.filters, additionalFilters] + ); + const { isProcessing, refetchFieldsExistenceInfo } = useExistingFieldsFetcher({ disableAutoFetching: stateService.creationOptions.disableFieldsExistenceAutoFetching, dataViews: searchMode === 'documents' && dataView ? [dataView] : [], query: querySubscriberResult.query, - filters: querySubscriberResult.filters, + filters, fromDate: querySubscriberResult.fromDate, toDate: querySubscriberResult.toDate, services, diff --git a/renovate.json b/renovate.json index 6f3b61c6e1b12..d013a49ea37bd 100644 --- a/renovate.json +++ b/renovate.json @@ -493,6 +493,15 @@ "labels": ["Team: Sec Eng Productivity", "release_note:skip", "backport:all-open"], "minimumReleaseAge": "7 days", "enabled": true + }, + { + "groupName": "@mswjs/http-middleware", + "matchPackageNames": ["@mswjs/http-middleware"], + "reviewers": ["team:kibana-cloud-security-posture"], + "matchBaseBranches": ["main"], + "labels": ["Team:Cloud Security", "release_note:skip", "backport:skip"], + "minimumReleaseAge": "7 days", + "enabled": true } ], "customManagers": [ diff --git a/src/dev/performance/run_performance_cli.ts b/src/dev/performance/run_performance_cli.ts index bb4192779207a..72f2bc46495a2 100644 --- a/src/dev/performance/run_performance_cli.ts +++ b/src/dev/performance/run_performance_cli.ts @@ -35,6 +35,19 @@ interface TestRunProps extends EsRunProps { kibanaInstallDir: string | undefined; } +interface JourneyTargetGroups { + [key: string]: string[]; +} + +const journeyTargetGroups: JourneyTargetGroups = { + kibanaStartAndLoad: ['login'], + crud: ['tags_listing_page', 'dashboard_listing_page'], + dashboard: ['ecommerce_dashboard', 'data_stress_test_lens', 'flight_dashboard'], + discover: ['many_fields_discover', 'many_fields_discover_esql'], + maps: ['ecommerce_dashboard_map_only'], + ml: ['aiops_log_rate_analysis', 'many_fields_transform', 'tsdb_logs_data_visualizer'], +}; + const readFilesRecursively = (dir: string, callback: Function) => { const files = fs.readdirSync(dir); files.forEach((file) => { @@ -48,6 +61,44 @@ const readFilesRecursively = (dir: string, callback: Function) => { }); }; +const getAllJourneys = (dir: string) => { + const journeys: Journey[] = []; + + readFilesRecursively(dir, (filePath: string) => + journeys.push({ + name: path.parse(filePath).name, + path: path.resolve(dir, filePath), + }) + ); + + return journeys; +}; + +const getJourneysToRun = ({ journeyPath, group }: { journeyPath?: string; group?: string }) => { + if (group && typeof group === 'string') { + if (!(group in journeyTargetGroups)) { + throw createFlagError(`Group '${group}' is not defined, try again`); + } + + const fileNames = journeyTargetGroups[group]; + const dir = path.resolve(REPO_ROOT, JOURNEY_BASE_PATH); + + return getAllJourneys(dir).filter((journey) => fileNames.includes(journey.name)); + } + + if (journeyPath && !fs.existsSync(journeyPath)) { + throw createFlagError('--journey-path must be an existing path'); + } + + if (journeyPath && fs.statSync(journeyPath).isFile()) { + return [{ name: path.parse(journeyPath).name, path: journeyPath }]; + } else { + // default dir is x-pack/performance/journeys_e2e + const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH); + return getAllJourneys(dir); + } +}; + async function startEs(props: EsRunProps) { const { procRunner, log, logsDir } = props; await procRunner.run('es', { @@ -115,29 +166,17 @@ run( const skipWarmup = flagsReader.boolean('skip-warmup'); const kibanaInstallDir = flagsReader.path('kibana-install-dir'); const journeyPath = flagsReader.path('journey-path'); + const group = flagsReader.string('group'); - if (kibanaInstallDir && !fs.existsSync(kibanaInstallDir)) { - throw createFlagError('--kibana-install-dir must be an existing directory'); + if (group && journeyPath) { + throw createFlagError('--group and --journeyPath cannot be used simultaneously'); } - if (journeyPath && !fs.existsSync(journeyPath)) { - throw createFlagError('--journey-path must be an existing path'); + if (kibanaInstallDir && !fs.existsSync(kibanaInstallDir)) { + throw createFlagError('--kibana-install-dir must be an existing directory'); } - const journeys: Journey[] = []; - - if (journeyPath && fs.statSync(journeyPath).isFile()) { - journeys.push({ name: path.parse(journeyPath).name, path: journeyPath }); - } else { - // default dir is x-pack/performance/journeys_e2e - const dir = journeyPath ?? path.resolve(REPO_ROOT, JOURNEY_BASE_PATH); - readFilesRecursively(dir, (filePath: string) => - journeys.push({ - name: path.parse(filePath).name, - path: path.resolve(dir, filePath), - }) - ); - } + const journeys = getJourneysToRun({ journeyPath, group }); if (journeys.length === 0) { throw new Error('No journeys found'); @@ -191,13 +230,14 @@ run( }, { flags: { - string: ['kibana-install-dir', 'journey-path'], + string: ['kibana-install-dir', 'journey-path', 'group'], boolean: ['skip-warmup'], help: ` --kibana-install-dir=dir Run Kibana from existing install directory instead of from source --journey-path=path Define path to performance journey or directory with multiple journeys that should be executed. '${JOURNEY_BASE_PATH}' is run by default --skip-warmup Journey will be executed without warmup (TEST phase only) + --group Run subset of journeys, defined in the specified group `, }, } diff --git a/src/plugins/console/public/application/containers/editor/utils/constants.ts b/src/plugins/console/public/application/containers/editor/utils/constants.ts index f97c9b4ef1d69..e6e5c1f622366 100644 --- a/src/plugins/console/public/application/containers/editor/utils/constants.ts +++ b/src/plugins/console/public/application/containers/editor/utils/constants.ts @@ -17,11 +17,7 @@ export const SELECTED_REQUESTS_CLASSNAME = 'console__monaco_editor__selectedRequ /* * CSS class names used for the styling of multiple-response status codes */ -export const PRIMARY_STATUS_BADGE_CLASSNAME = 'monaco__status_badge--primary'; -export const SUCCESS_STATUS_BADGE_CLASSNAME = 'monaco__status_badge--success'; -export const DEFAULT_STATUS_BADGE_CLASSNAME = 'monaco__status_badge--default'; -export const WARNING_STATUS_BADGE_CLASSNAME = 'monaco__status_badge--warning'; -export const DANGER_STATUS_BADGE_CLASSNAME = 'monaco__status_badge--danger'; +export const STATUS_CODE_LINE_CLASSNAME = 'monaco__status_code_line'; export const whitespacesRegex = /\s+/; export const newLineRegex = /\n/; diff --git a/src/plugins/console/public/application/containers/editor/utils/index.ts b/src/plugins/console/public/application/containers/editor/utils/index.ts index 1e58b3dc7fd71..d5c10cae6bdb3 100644 --- a/src/plugins/console/public/application/containers/editor/utils/index.ts +++ b/src/plugins/console/public/application/containers/editor/utils/index.ts @@ -10,11 +10,7 @@ export { AutocompleteType, SELECTED_REQUESTS_CLASSNAME, - SUCCESS_STATUS_BADGE_CLASSNAME, - WARNING_STATUS_BADGE_CLASSNAME, - PRIMARY_STATUS_BADGE_CLASSNAME, - DEFAULT_STATUS_BADGE_CLASSNAME, - DANGER_STATUS_BADGE_CLASSNAME, + STATUS_CODE_LINE_CLASSNAME, } from './constants'; export { getRequestStartLineNumber, diff --git a/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.test.ts b/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.test.ts index 8ea919257cb26..b93dd4d0a211b 100644 --- a/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.test.ts +++ b/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.test.ts @@ -8,13 +8,17 @@ */ import { getStatusCodeDecorations } from './status_code_decoration_utils'; -import { - SUCCESS_STATUS_BADGE_CLASSNAME, - WARNING_STATUS_BADGE_CLASSNAME, - DANGER_STATUS_BADGE_CLASSNAME, -} from './constants'; +import { STATUS_CODE_LINE_CLASSNAME } from './constants'; import { RequestResult } from '../../../hooks/use_send_current_request/send_request'; +const SUCCESS_STATUS_CODE_CLASSNAME = `${STATUS_CODE_LINE_CLASSNAME}--success`; +const WARNING_STATUS_CODE_CLASSNAME = `${STATUS_CODE_LINE_CLASSNAME}--warning`; +const DANGER_STATUS_CODE_CLASSNAME = `${STATUS_CODE_LINE_CLASSNAME}--danger`; + +const SUCCESS_STATUS_CODE_LINE_CLASSNAME = `${STATUS_CODE_LINE_CLASSNAME}_number--success`; +const WARNING_STATUS_CODE_LINE_CLASSNAME = `${STATUS_CODE_LINE_CLASSNAME}_number--warning`; +const DANGER_STATUS_CODE_LINE_CLASSNAME = `${STATUS_CODE_LINE_CLASSNAME}_number--danger`; + describe('getStatusCodeDecorations', () => { it('correctly returns all decorations on full data', () => { // Sample multiple-response data returned from ES: @@ -91,108 +95,45 @@ describe('getStatusCodeDecorations', () => { const EXPECTED_DECORATIONS = [ { range: { - endColumn: 21, + endColumn: 1, endLineNumber: 1, - startColumn: 15, + startColumn: 1, startLineNumber: 1, }, options: { - inlineClassName: SUCCESS_STATUS_BADGE_CLASSNAME, + isWholeLine: true, + blockClassName: SUCCESS_STATUS_CODE_CLASSNAME, + marginClassName: SUCCESS_STATUS_CODE_LINE_CLASSNAME, }, }, { range: { - endColumn: 28, + endColumn: 1, endLineNumber: 12, - startColumn: 13, + startColumn: 1, startLineNumber: 12, }, options: { - inlineClassName: WARNING_STATUS_BADGE_CLASSNAME, + isWholeLine: true, + blockClassName: WARNING_STATUS_CODE_CLASSNAME, + marginClassName: WARNING_STATUS_CODE_LINE_CLASSNAME, }, }, { range: { - endColumn: 47, + endColumn: 1, endLineNumber: 18, - startColumn: 22, + startColumn: 1, startLineNumber: 18, }, options: { - inlineClassName: DANGER_STATUS_BADGE_CLASSNAME, + isWholeLine: true, + blockClassName: DANGER_STATUS_CODE_CLASSNAME, + marginClassName: DANGER_STATUS_CODE_LINE_CLASSNAME, }, }, ]; expect(getStatusCodeDecorations(SAMPLE_COMPLETE_DATA)).toEqual(EXPECTED_DECORATIONS); }); - - it('only returns decorations for data with complete status code and text', () => { - // This sample data is same as in previous test but some of it has incomplete status code or status text - const SAMPLE_INCOMPLETE_DATA: RequestResult[] = [ - { - response: { - timeMs: 50, - // @ts-ignore - statusCode: undefined, - statusText: 'OK', - contentType: 'application/json', - value: - '# GET _search OK\n{\n"took": 1,\n"timed_out": false,\n"hits": {\n"total": {\n"value": 0,\n"relation": "eq"\n}\n}\n}', - }, - request: { - data: '', - method: 'GET', - path: '_search', - }, - }, - { - response: { - timeMs: 22, - statusCode: 400, - statusText: 'Bad Request', - contentType: 'application/json', - value: '# GET _test 400 Bad Request\n{\n"error": {\n"root_cause": [],\n"status": 400\n}', - }, - request: { - data: '', - method: 'GET', - path: '_test', - }, - }, - { - response: { - timeMs: 23, - // @ts-ignore - statusCode: undefined, - // @ts-ignore - statusText: undefined, - contentType: 'application/json', - value: '# PUT /library/_bulk\n{\n"error": {\n"root_cause": [],\n"status": 500\n}', - }, - request: { - data: '', - method: 'PUT', - path: '/library/_bulk?refresh', - }, - }, - ]; - - // Only the second response has complete status code and text - const EXPECTED_DECORATIONS = [ - { - range: { - endColumn: 28, - endLineNumber: 12, - startColumn: 13, - startLineNumber: 12, - }, - options: { - inlineClassName: WARNING_STATUS_BADGE_CLASSNAME, - }, - }, - ]; - - expect(getStatusCodeDecorations(SAMPLE_INCOMPLETE_DATA)).toEqual(EXPECTED_DECORATIONS); - }); }); diff --git a/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.ts b/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.ts index 01b87abb2fdf5..daa7052223494 100644 --- a/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.ts +++ b/src/plugins/console/public/application/containers/editor/utils/status_code_decoration_utils.ts @@ -9,28 +9,22 @@ import { monaco } from '@kbn/monaco'; import { RequestResult } from '../../../hooks/use_send_current_request/send_request'; -import { - DEFAULT_STATUS_BADGE_CLASSNAME, - SUCCESS_STATUS_BADGE_CLASSNAME, - PRIMARY_STATUS_BADGE_CLASSNAME, - WARNING_STATUS_BADGE_CLASSNAME, - DANGER_STATUS_BADGE_CLASSNAME, -} from './constants'; +import { STATUS_CODE_LINE_CLASSNAME } from './constants'; -const getStatusCodeClassName = (statusCode: number) => { +const getStatusCodeClassNameSuffix = (statusCode: number) => { if (statusCode <= 199) { - return DEFAULT_STATUS_BADGE_CLASSNAME; + return '--default'; } if (statusCode <= 299) { - return SUCCESS_STATUS_BADGE_CLASSNAME; + return '--success'; } if (statusCode <= 399) { - return PRIMARY_STATUS_BADGE_CLASSNAME; + return '--primary'; } if (statusCode <= 499) { - return WARNING_STATUS_BADGE_CLASSNAME; + return '--warning'; } - return DANGER_STATUS_BADGE_CLASSNAME; + return '--danger'; }; export const getStatusCodeDecorations = (data: RequestResult[]) => { @@ -39,25 +33,21 @@ export const getStatusCodeDecorations = (data: RequestResult[]) => { data.forEach(({ response }) => { if (response?.value) { - const totalStatus = - response.statusCode && response.statusText - ? response.statusCode + ' ' + response.statusText - : ''; - const startColumn = (response.value as string).indexOf(totalStatus) + 1; - if (totalStatus && startColumn !== 0) { - const range = { - startLineNumber: lastResponseEndLine + 1, - startColumn, - endLineNumber: lastResponseEndLine + 1, - endColumn: startColumn + totalStatus.length, - }; - decorations.push({ - range, - options: { - inlineClassName: getStatusCodeClassName(response.statusCode), - }, - }); - } + const range = { + startLineNumber: lastResponseEndLine + 1, + startColumn: 1, + endLineNumber: lastResponseEndLine + 1, + endColumn: 1, // It doesn't matter what endColumn we set as the decoration will be applied to the whole line + }; + const classNameSuffix = getStatusCodeClassNameSuffix(response.statusCode); + decorations.push({ + range, + options: { + isWholeLine: true, + blockClassName: `${STATUS_CODE_LINE_CLASSNAME}${classNameSuffix}`, + marginClassName: `${STATUS_CODE_LINE_CLASSNAME}_number${classNameSuffix}`, + }, + }); lastResponseEndLine += (response.value as string).split(/\\n|\n/).length; } }); diff --git a/src/plugins/console/public/application/hooks/use_send_current_request/send_request.ts b/src/plugins/console/public/application/hooks/use_send_current_request/send_request.ts index 101831300cb89..40f440eccd7ce 100644 --- a/src/plugins/console/public/application/hooks/use_send_current_request/send_request.ts +++ b/src/plugins/console/public/application/hooks/use_send_current_request/send_request.ts @@ -130,7 +130,7 @@ export function sendRequest(args: RequestArgs): Promise { if (isMultiRequest) { const lineNumber = req.lineNumber ? `${req.lineNumber}: ` : ''; - value = `# ${lineNumber}${req.method} ${req.url} ${statusCode} ${statusText}\n${value}`; + value = `# ${lineNumber}${req.method} ${req.url} [${statusCode} ${statusText}]\n${value}`; } results.push({ @@ -164,7 +164,8 @@ export function sendRequest(args: RequestArgs): Promise { } if (isMultiRequest) { - value = `# ${req.method} ${req.url} ${statusCode} ${statusText}\n${value}`; + const lineNumber = req.lineNumber ? `${req.lineNumber}: ` : ''; + value = `# ${lineNumber}${req.method} ${req.url} [${statusCode} ${statusText}]\n${value}`; } const result = { diff --git a/src/plugins/console/public/styles/_app.scss b/src/plugins/console/public/styles/_app.scss index 74367c32a468e..51fb701a57b46 100644 --- a/src/plugins/console/public/styles/_app.scss +++ b/src/plugins/console/public/styles/_app.scss @@ -203,29 +203,44 @@ max-width: 100%; } -.monaco__status_badge--primary { - @extend %monaco__status_badge; - background-color: $euiColorVis1; +.monaco__status_code_line--primary { + background-color: transparentize($euiColorVis1, .9); } -.monaco__status_badge--success { - @extend %monaco__status_badge; - background-color: $euiColorSuccess; +.monaco__status_code_line_number--primary { + background-color: transparentize($euiColorVis1, .5); } -.monaco__status_badge--default { - @extend %monaco__status_badge; - background-color: $euiColorLightShade; +.monaco__status_code_line--success { + background-color: transparentize($euiColorSuccess, .9); } -.monaco__status_badge--warning { - @extend %monaco__status_badge; - background-color: $euiColorWarning; +.monaco__status_code_line_number--success { + background-color: transparentize($euiColorSuccess, .5); } -.monaco__status_badge--danger { - @extend %monaco__status_badge; - background-color: $euiColorDanger; +.monaco__status_code_line--default { + background-color: transparentize($euiColorLightShade, .9); +} + +.monaco__status_code_line_number--default { + background-color: transparentize($euiColorLightShade, .5); +} + +.monaco__status_code_line--warning { + background-color: transparentize($euiColorWarning, .9); +} + +.monaco__status_code_line_number--warning { + background-color: transparentize($euiColorWarning, .5); +} + +.monaco__status_code_line--danger { + background-color: transparentize($euiColorDanger, .9); +} + +.monaco__status_code_line_number--danger { + background-color: transparentize($euiColorDanger, .5); } /* diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/esql.query.json b/src/plugins/console/server/lib/spec_definitions/json/generated/esql.query.json index f109f849be68c..4180fc878a0ba 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/esql.query.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/esql.query.json @@ -5,7 +5,16 @@ "filter_path": [], "human": "__flag__", "pretty": "__flag__", - "format": "", + "format": [ + "csv", + "json", + "tsv", + "txt", + "yaml", + "cbor", + "smile", + "arrow" + ], "delimiter": "" }, "methods": [ diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/indices.resolve_index.json b/src/plugins/console/server/lib/spec_definitions/json/generated/indices.resolve_index.json index 77a3fc9cc18a2..790e5ecd171f1 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/indices.resolve_index.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/indices.resolve_index.json @@ -11,7 +11,9 @@ "closed", "hidden", "none" - ] + ], + "ignore_unavailable": "__flag__", + "allow_no_indices": "__flag__" }, "methods": [ "GET" diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.delete_geoip_database.json b/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.delete_geoip_database.json new file mode 100644 index 0000000000000..109dfce014012 --- /dev/null +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.delete_geoip_database.json @@ -0,0 +1,31 @@ +{ + "ingest.delete_geoip_database": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__", + "master_timeout": [ + "30s", + "-1", + "0" + ], + "timeout": [ + "30s", + "-1", + "0" + ] + }, + "methods": [ + "DELETE" + ], + "patterns": [ + "_ingest/geoip/database/{id}" + ], + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-geoip-database-api.html", + "availability": { + "stack": true, + "serverless": false + } + } +} diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.get_geoip_database.json b/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.get_geoip_database.json new file mode 100644 index 0000000000000..e57b36f09cbf7 --- /dev/null +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.get_geoip_database.json @@ -0,0 +1,27 @@ +{ + "ingest.get_geoip_database": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__", + "master_timeout": [ + "30s", + "-1", + "0" + ] + }, + "methods": [ + "GET" + ], + "patterns": [ + "_ingest/geoip/database", + "_ingest/geoip/database/{id}" + ], + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-geoip-database-api.html", + "availability": { + "stack": true, + "serverless": false + } + } +} diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.put_geoip_database.json b/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.put_geoip_database.json new file mode 100644 index 0000000000000..675c01239af38 --- /dev/null +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/ingest.put_geoip_database.json @@ -0,0 +1,31 @@ +{ + "ingest.put_geoip_database": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__", + "master_timeout": [ + "30s", + "-1", + "0" + ], + "timeout": [ + "30s", + "-1", + "0" + ] + }, + "methods": [ + "PUT" + ], + "patterns": [ + "_ingest/geoip/database/{id}" + ], + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-geoip-database-api.html", + "availability": { + "stack": true, + "serverless": false + } + } +} diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/msearch.json b/src/plugins/console/server/lib/spec_definitions/json/generated/msearch.json index 771c603c87924..58308e10f4fbe 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/msearch.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/msearch.json @@ -16,6 +16,7 @@ ], "ignore_throttled": "__flag__", "ignore_unavailable": "__flag__", + "include_named_queries_score": "__flag__", "max_concurrent_searches": "", "max_concurrent_shard_requests": [ "5" diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.delete.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.delete_rule.json similarity index 93% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.delete.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.delete_rule.json index eea5d085b03bc..4255ae1672064 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.delete.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.delete_rule.json @@ -1,5 +1,5 @@ { - "query_rule.delete": { + "query_rules.delete_rule": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.delete.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.delete_ruleset.json similarity index 92% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.delete.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.delete_ruleset.json index c0e564ec5c212..d4f34e7d76716 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.delete.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.delete_ruleset.json @@ -1,5 +1,5 @@ { - "query_ruleset.delete": { + "query_rules.delete_ruleset": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.get.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.get_rule.json similarity index 94% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.get.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.get_rule.json index 27564d1fe7b16..fdd424c2a2d1b 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.get.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.get_rule.json @@ -1,5 +1,5 @@ { - "query_rule.get": { + "query_rules.get_rule": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.get.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.get_ruleset.json similarity index 93% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.get.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.get_ruleset.json index 424b749bc3a19..23cff910d815e 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.get.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.get_ruleset.json @@ -1,5 +1,5 @@ { - "query_ruleset.get": { + "query_rules.get_ruleset": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.list.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.list_rulesets.json similarity index 93% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.list.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.list_rulesets.json index 3213cdbe9d6d6..c20f8050c5c91 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.list.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.list_rulesets.json @@ -1,5 +1,5 @@ { - "query_ruleset.list": { + "query_rules.list_rulesets": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.put.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.put_rule.json similarity index 94% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.put.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.put_rule.json index 346eeba09e6ed..2653a15dab650 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_rule.put.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.put_rule.json @@ -1,5 +1,5 @@ { - "query_rule.put": { + "query_rules.put_rule": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.put.json b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.put_ruleset.json similarity index 93% rename from src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.put.json rename to src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.put_ruleset.json index 924e449ab29d3..a4c0ad3c72d94 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/query_ruleset.put.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/query_rules.put_ruleset.json @@ -1,5 +1,5 @@ { - "query_ruleset.put": { + "query_rules.put_ruleset": { "url_params": { "error_trace": "__flag__", "filter_path": [], diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/search.json b/src/plugins/console/server/lib/spec_definitions/json/generated/search.json index dd41962508677..ffa3709b9ae3f 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/search.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/search.json @@ -29,6 +29,7 @@ "explain": "__flag__", "ignore_throttled": "__flag__", "ignore_unavailable": "__flag__", + "include_named_queries_score": "__flag__", "lenient": "__flag__", "max_concurrent_shard_requests": [ "5" diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/security.bulk_delete_role.json b/src/plugins/console/server/lib/spec_definitions/json/generated/security.bulk_delete_role.json new file mode 100644 index 0000000000000..cbc71a4354533 --- /dev/null +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/security.bulk_delete_role.json @@ -0,0 +1,26 @@ +{ + "security.bulk_delete_role": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__", + "refresh": [ + "true", + "false", + "wait_for" + ] + }, + "methods": [ + "DELETE" + ], + "patterns": [ + "_security/role" + ], + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-bulk-delete-role.html", + "availability": { + "stack": true, + "serverless": false + } + } +} diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/security.bulk_put_role.json b/src/plugins/console/server/lib/spec_definitions/json/generated/security.bulk_put_role.json new file mode 100644 index 0000000000000..a31328b512882 --- /dev/null +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/security.bulk_put_role.json @@ -0,0 +1,26 @@ +{ + "security.bulk_put_role": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__", + "refresh": [ + "true", + "false", + "wait_for" + ] + }, + "methods": [ + "POST" + ], + "patterns": [ + "_security/role" + ], + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-bulk-put-role.html", + "availability": { + "stack": true, + "serverless": false + } + } +} diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_role.json b/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_role.json new file mode 100644 index 0000000000000..7648662f973c0 --- /dev/null +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_role.json @@ -0,0 +1,22 @@ +{ + "security.query_role": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__" + }, + "methods": [ + "GET", + "POST" + ], + "patterns": [ + "_security/_query/role" + ], + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-role.html", + "availability": { + "stack": true, + "serverless": false + } + } +} diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_user.json b/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_user.json index 4b8f22799b4bc..811d5c907173b 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_user.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/security.query_user.json @@ -1,5 +1,12 @@ { "security.query_user": { + "url_params": { + "error_trace": "__flag__", + "filter_path": [], + "human": "__flag__", + "pretty": "__flag__", + "with_profile_uid": "__flag__" + }, "methods": [ "GET", "POST" diff --git a/src/plugins/console/server/lib/spec_definitions/json/generated/update_by_query.json b/src/plugins/console/server/lib/spec_definitions/json/generated/update_by_query.json index 787d36fdb2ec6..2d382decb9c2e 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/generated/update_by_query.json +++ b/src/plugins/console/server/lib/spec_definitions/json/generated/update_by_query.json @@ -30,6 +30,7 @@ "max_docs": "", "pipeline": "", "preference": "", + "q": "", "refresh": "__flag__", "request_cache": "__flag__", "requests_per_second": [ diff --git a/src/plugins/console/server/lib/spec_definitions/json/overrides/cluster.health.json b/src/plugins/console/server/lib/spec_definitions/json/overrides/cluster.health.json index 949b897b29ff4..1c432db8e00e8 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/overrides/cluster.health.json +++ b/src/plugins/console/server/lib/spec_definitions/json/overrides/cluster.health.json @@ -1,38 +1,10 @@ { "cluster.health": { "url_params": { - "expand_wildcards": [ - "open", - "closed", - "hidden", - "none", - "all" - ], - "level": [ - "cluster", - "indices", - "shards" - ], - "local": "__flag__", "master_timeout": "", "timeout": "", "wait_for_active_shards": "", - "wait_for_nodes": "", - "wait_for_events": [ - "immediate", - "urgent", - "high", - "normal", - "low", - "languid" - ], - "wait_for_no_relocating_shards": "__flag__", - "wait_for_no_initializing_shards": "__flag__", - "wait_for_status": [ - "green", - "yellow", - "red" - ] + "wait_for_nodes": "" } } } diff --git a/src/plugins/console/server/lib/spec_definitions/json/overrides/esql.query.json b/src/plugins/console/server/lib/spec_definitions/json/overrides/esql.query.json index 2dd1f9b50ee3b..0947840158818 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/overrides/esql.query.json +++ b/src/plugins/console/server/lib/spec_definitions/json/overrides/esql.query.json @@ -5,21 +5,6 @@ "locale": "", "params": [], "query": "" - }, - "url_params": { - "format": [ - "cbor", - "csv", - "json", - "smile", - "txt", - "tsv", - "yaml" - ], - "drop_null_columns": [ - "false", - "true" - ] } } } diff --git a/src/plugins/console/server/lib/spec_definitions/json/overrides/query_ruleset.put.json b/src/plugins/console/server/lib/spec_definitions/json/overrides/query_rules.put_ruleset.json similarity index 93% rename from src/plugins/console/server/lib/spec_definitions/json/overrides/query_ruleset.put.json rename to src/plugins/console/server/lib/spec_definitions/json/overrides/query_rules.put_ruleset.json index 9b50197db9689..21d598b0cdf7b 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/overrides/query_ruleset.put.json +++ b/src/plugins/console/server/lib/spec_definitions/json/overrides/query_rules.put_ruleset.json @@ -1,5 +1,5 @@ { - "query_ruleset.put": { + "query_rules.put_ruleset": { "data_autocomplete_rules": { "rules": [{ "rule_id": "", diff --git a/src/plugins/console/server/lib/spec_definitions/json/overrides/search.json b/src/plugins/console/server/lib/spec_definitions/json/overrides/search.json deleted file mode 100644 index 1028422b303f2..0000000000000 --- a/src/plugins/console/server/lib/spec_definitions/json/overrides/search.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "search": { - "url_params": { - "error_trace": true - } - } -} diff --git a/src/plugins/console/server/lib/spec_definitions/json/overrides/xpack.info.json b/src/plugins/console/server/lib/spec_definitions/json/overrides/xpack.info.json index fff8fda8328f9..be604c799cf26 100644 --- a/src/plugins/console/server/lib/spec_definitions/json/overrides/xpack.info.json +++ b/src/plugins/console/server/lib/spec_definitions/json/overrides/xpack.info.json @@ -5,8 +5,7 @@ "build", "license", "features" - ], - "human": "__flag__" + ] } } } diff --git a/src/plugins/discover/public/application/main/components/document_explorer_callout/document_explorer_update_callout.test.tsx b/src/plugins/discover/public/application/main/components/document_explorer_callout/document_explorer_update_callout.test.tsx deleted file mode 100644 index 52461bc307684..0000000000000 --- a/src/plugins/discover/public/application/main/components/document_explorer_callout/document_explorer_update_callout.test.tsx +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React from 'react'; -import { mountWithIntl, findTestSubject } from '@kbn/test-jest-helpers'; -import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; -import { - CALLOUT_STATE_KEY, - DocumentExplorerUpdateCallout, -} from './document_explorer_update_callout'; -import { LocalStorageMock } from '../../../../__mocks__/local_storage_mock'; -import { DiscoverServices } from '../../../../build_services'; -import { discoverServiceMock } from '../../../../__mocks__/services'; -import { DiscoverTourProvider } from '../../../../components/discover_tour'; -import { getDiscoverStateMock } from '../../../../__mocks__/discover_state.mock'; -import { DiscoverMainProvider } from '../../state_management/discover_state_provider'; - -const defaultServices = { - ...discoverServiceMock, - capabilities: { ...discoverServiceMock.capabilities, advancedSettings: { save: true } }, - storage: new LocalStorageMock({ [CALLOUT_STATE_KEY]: false }), -} as unknown as DiscoverServices; - -const mount = (services: DiscoverServices) => { - return mountWithIntl( - - - - ); -}; - -describe('Document Explorer Update callout', () => { - it('should render callout', () => { - const result = mount(defaultServices); - - expect(result.find('.dscDocumentExplorerCallout').exists()).toBeTruthy(); - }); - - it('should not render callout for user without permissions', () => { - const services = { - ...defaultServices, - capabilities: { advancedSettings: { save: false } }, - } as unknown as DiscoverServices; - const result = mount(services); - - expect(result.find('.dscDocumentExplorerCallout').exists()).toBeFalsy(); - }); - - it('should not render callout of it was closed', () => { - const services = { - ...defaultServices, - storage: new LocalStorageMock({ [CALLOUT_STATE_KEY]: true }), - } as unknown as DiscoverServices; - const result = mount(services); - - expect(result.find('.dscDocumentExplorerCallout').exists()).toBeFalsy(); - }); - - it('should start a tour when the button is clicked', () => { - const result = mountWithIntl( - - - - - - - - ); - - expect(result.find({ isStepOpen: true })).toHaveLength(0); - findTestSubject(result, 'discoverTakeTourButton').simulate('click'); - expect(result.find({ isStepOpen: true })).toHaveLength(1); - }); -}); diff --git a/src/plugins/discover/public/application/main/components/document_explorer_callout/document_explorer_update_callout.tsx b/src/plugins/discover/public/application/main/components/document_explorer_callout/document_explorer_update_callout.tsx deleted file mode 100644 index a9a2065046b3d..0000000000000 --- a/src/plugins/discover/public/application/main/components/document_explorer_callout/document_explorer_update_callout.tsx +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React, { useCallback, useState } from 'react'; -import './document_explorer_callout.scss'; -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; -import { - EuiButton, - EuiButtonEmpty, - EuiButtonIcon, - EuiCallOut, - EuiFlexGroup, - EuiFlexItem, -} from '@elastic/eui'; -import { Storage } from '@kbn/kibana-utils-plugin/public'; -import { useDiscoverServices } from '../../../../hooks/use_discover_services'; -import { useDiscoverTourContext } from '../../../../components/discover_tour'; - -export const CALLOUT_STATE_KEY = 'discover:docExplorerUpdateCalloutClosed'; - -const getStoredCalloutState = (storage: Storage): boolean => { - const calloutClosed = storage.get(CALLOUT_STATE_KEY); - return Boolean(calloutClosed); -}; -const updateStoredCalloutState = (newState: boolean, storage: Storage) => { - storage.set(CALLOUT_STATE_KEY, newState); -}; - -/** - * The callout that's displayed when Document explorer is enabled - */ -export const DocumentExplorerUpdateCallout = () => { - const { storage, capabilities } = useDiscoverServices(); - const [calloutClosed, setCalloutClosed] = useState(getStoredCalloutState(storage)); - const { onStartTour } = useDiscoverTourContext(); - - const onCloseCallout = useCallback(() => { - updateStoredCalloutState(true, storage); - setCalloutClosed(true); - }, [storage]); - - if (calloutClosed || !capabilities.advancedSettings.save) { - return null; - } - - return ( - } - iconType="tableDensityNormal" - heading="h3" - size="s" - > -

- -

- - - - - - - - - - - - - - ); -}; - -function CalloutTitle({ onCloseCallout }: { onCloseCallout: () => void }) { - return ( - - - - - - - - - ); -} diff --git a/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx b/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx index 6d739df33b8d1..4d8fd0fbccb95 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx @@ -58,11 +58,6 @@ import { DiscoverStateContainer } from '../../state_management/discover_state'; import { useDataState } from '../../hooks/use_data_state'; import { DocTableInfinite } from '../../../../components/doc_table/doc_table_infinite'; import { DocumentExplorerCallout } from '../document_explorer_callout'; -import { DocumentExplorerUpdateCallout } from '../document_explorer_callout/document_explorer_update_callout'; -import { - DISCOVER_TOUR_STEP_ANCHOR_IDS, - DiscoverTourProvider, -} from '../../../../components/discover_tour'; import { getMaxAllowedSampleSize, getAllowedSampleSize, @@ -90,8 +85,6 @@ const progressStyle = css` z-index: 2; `; -const TOUR_STEPS = { expandButton: DISCOVER_TOUR_STEP_ANCHOR_IDS.expandDocument }; - const DocTableInfiniteMemoized = React.memo(DocTableInfinite); const DiscoverGridMemoized = React.memo(DiscoverGrid); @@ -340,18 +333,6 @@ function DiscoverDocumentsComponent({ [currentColumns, documents?.esqlQueryColumns, documentState.interceptedWarnings] ); - const gridAnnouncementCallout = useMemo(() => { - if (hideAnnouncements || isLegacy) { - return null; - } - - return !isEsqlMode ? ( - - - - ) : null; - }, [hideAnnouncements, isLegacy, isEsqlMode]); - const loadingIndicator = useMemo( () => isDataLoading ? ( @@ -373,12 +354,11 @@ function DiscoverDocumentsComponent({ bottomSection: ( <> {callouts} - {gridAnnouncementCallout} {loadingIndicator} ), }), - [viewModeToggle, callouts, gridAnnouncementCallout, loadingIndicator] + [viewModeToggle, callouts, loadingIndicator] ); if (isDataViewLoading || (isEmptyDataResult && isDataLoading)) { @@ -486,7 +466,6 @@ function DiscoverDocumentsComponent({ services={services} totalHits={totalHits} onFetchMoreRecords={onFetchMoreRecords} - componentsTourSteps={TOUR_STEPS} externalCustomRenderers={cellRenderers} customGridColumnsConfiguration={customGridColumnsConfiguration} rowAdditionalLeadingControls={rowAdditionalLeadingControls} diff --git a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx index 97e1a34f85caf..82403ad38c710 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx @@ -103,6 +103,8 @@ export function DiscoverLayout({ stateContainer }: DiscoverLayoutProps) { state.dataView!, state.isDataViewLoading, ]); + const customFilters = useInternalStateSelector((state) => state.customFilters); + const dataState: DataMainMsg = useDataState(main$); const savedSearch = useSavedSearchInitial(); @@ -401,6 +403,7 @@ export function DiscoverLayout({ stateContainer }: DiscoverLayoutProps) { onFieldEdited={onFieldEdited} onDataViewCreated={stateContainer.actions.onDataViewCreated} sidebarToggleState$={sidebarToggleState$} + additionalFilters={customFilters} /> } mainPanel={ diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx index e332da22f4776..80a3b9d412c76 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx @@ -23,11 +23,11 @@ import { FieldsGroupNames, } from '@kbn/unified-field-list'; import { calcFieldCounts } from '@kbn/discover-utils/src/utils/calc_field_counts'; +import { Filter } from '@kbn/es-query'; import { PLUGIN_ID } from '../../../../../common'; import { useDiscoverServices } from '../../../../hooks/use_discover_services'; import { DataDocuments$ } from '../../state_management/discover_data_state_container'; import { FetchStatus, SidebarToggleState } from '../../../types'; -import { DISCOVER_TOUR_STEP_ANCHOR_IDS } from '../../../../components/discover_tour'; import { discoverSidebarReducer, getInitialState, @@ -48,11 +48,6 @@ const getCreationOptions: UnifiedFieldListSidebarContainerProps['getCreationOpti showSidebarToggleButton: true, disableFieldsExistenceAutoFetching: true, buttonAddFieldVariant: 'toolbar', - buttonPropsToTriggerFlyout: { - contentProps: { - id: DISCOVER_TOUR_STEP_ANCHOR_IDS.addFields, - }, - }, buttonAddFieldToWorkspaceProps: { 'aria-label': i18n.translate('discover.fieldChooser.discoverField.addFieldTooltip', { defaultMessage: 'Add field as column', @@ -133,6 +128,10 @@ export interface DiscoverSidebarResponsiveProps { fieldListVariant?: UnifiedFieldListSidebarContainerProps['variant']; sidebarToggleState$: BehaviorSubject; + /** + * Custom filters to apply for the field list, ex: namespace custom filter + */ + additionalFilters?: Filter[]; } /** @@ -159,6 +158,7 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps) onAddField, onRemoveField, sidebarToggleState$, + additionalFilters, } = props; const [sidebarState, dispatchSidebarStateAction] = useReducer( discoverSidebarReducer, @@ -389,6 +389,7 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps) onFieldEdited={onFieldEdited} prependInFlyout={prependDataViewPickerForMobile} additionalFieldGroups={additionalFieldGroups} + additionalFilters={additionalFilters} /> ) : null} diff --git a/src/plugins/discover/public/assets/discover_tour/add_fields.gif b/src/plugins/discover/public/assets/discover_tour/add_fields.gif deleted file mode 100644 index c955a9aa99e08..0000000000000 Binary files a/src/plugins/discover/public/assets/discover_tour/add_fields.gif and /dev/null differ diff --git a/src/plugins/discover/public/assets/discover_tour/expand.gif b/src/plugins/discover/public/assets/discover_tour/expand.gif deleted file mode 100644 index 7131fed839478..0000000000000 Binary files a/src/plugins/discover/public/assets/discover_tour/expand.gif and /dev/null differ diff --git a/src/plugins/discover/public/assets/discover_tour/reorder_columns.gif b/src/plugins/discover/public/assets/discover_tour/reorder_columns.gif deleted file mode 100644 index d3aeedb513c1e..0000000000000 Binary files a/src/plugins/discover/public/assets/discover_tour/reorder_columns.gif and /dev/null differ diff --git a/src/plugins/discover/public/assets/discover_tour/rows_per_line.gif b/src/plugins/discover/public/assets/discover_tour/rows_per_line.gif deleted file mode 100644 index 66033d03d8fd2..0000000000000 Binary files a/src/plugins/discover/public/assets/discover_tour/rows_per_line.gif and /dev/null differ diff --git a/src/plugins/discover/public/assets/discover_tour/sort.gif b/src/plugins/discover/public/assets/discover_tour/sort.gif deleted file mode 100644 index 6d22b947a206f..0000000000000 Binary files a/src/plugins/discover/public/assets/discover_tour/sort.gif and /dev/null differ diff --git a/src/plugins/discover/public/components/data_types/logs/popover_chip.tsx b/src/plugins/discover/public/components/data_types/logs/popover_chip.tsx index e5244a60eedac..e84fcca52d627 100644 --- a/src/plugins/discover/public/components/data_types/logs/popover_chip.tsx +++ b/src/plugins/discover/public/components/data_types/logs/popover_chip.tsx @@ -30,7 +30,11 @@ const DataTablePopoverCellValue = dynamic( () => import('@kbn/unified-data-table/src/components/data_table_cell_value') ); -interface ChipWithPopoverProps { +type ChipWithPopoverChildrenType = (props: { + content: string; +}) => React.ReactNode | React.ReactNode; + +export interface ChipWithPopoverProps { /** * ECS mapping for the key */ @@ -42,6 +46,7 @@ interface ChipWithPopoverProps { dataTestSubj?: string; leftSideIcon?: React.ReactNode; rightSideIcon?: EuiBadgeProps['iconType']; + children?: ChipWithPopoverChildrenType; } export function ChipWithPopover({ @@ -50,6 +55,7 @@ export function ChipWithPopover({ dataTestSubj = `dataTablePopoverChip_${property}`, leftSideIcon, rightSideIcon, + children, }: ChipWithPopoverProps) { return ( )} - /> + > + {children} + ); } @@ -89,9 +97,10 @@ interface ChipPopoverProps { handleChipClickAriaLabel: string; chipCss: SerializedStyles; }) => ReactElement; + children?: ChipWithPopoverChildrenType; } -export function ChipPopover({ property, text, renderChip }: ChipPopoverProps) { +export function ChipPopover({ property, text, renderChip, children }: ChipPopoverProps) { const xsFontSize = useEuiFontSize('xs').fontSize; const [isPopoverOpen, setIsPopoverOpen] = useState(false); @@ -131,7 +140,8 @@ export function ChipPopover({ property, text, renderChip }: ChipPopoverProps) {
- {property} {text} + {property}{' '} + {typeof children === 'function' ? children({ content: text }) : text}
diff --git a/src/plugins/discover/public/components/data_types/logs/service_name_chip_with_popover.tsx b/src/plugins/discover/public/components/data_types/logs/service_name_chip_with_popover.tsx new file mode 100644 index 0000000000000..ecb456017c7a9 --- /dev/null +++ b/src/plugins/discover/public/components/data_types/logs/service_name_chip_with_popover.tsx @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { getRouterLinkProps } from '@kbn/router-utils'; +import { EuiLink } from '@elastic/eui'; +import { OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE } from '@kbn/management-settings-ids'; +import { type ChipWithPopoverProps, ChipWithPopover } from './popover_chip'; +import { useDiscoverServices } from '../../../hooks/use_discover_services'; + +const SERVICE_ENTITY_LOCATOR = 'SERVICE_ENTITY_LOCATOR'; + +export function ServiceNameChipWithPopover(props: ChipWithPopoverProps) { + const { share, core } = useDiscoverServices(); + const canViewApm = core.application.capabilities.apm?.show || false; + const isEntityCentricExperienceSettingEnabled = canViewApm + ? core.uiSettings.get(OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE) + : false; + const urlService = share?.url; + + const apmLinkToServiceEntityLocator = urlService?.locators.get<{ serviceName: string }>( + SERVICE_ENTITY_LOCATOR + ); + const href = apmLinkToServiceEntityLocator?.getRedirectUrl({ + serviceName: props.text, + }); + + const routeLinkProps = href + ? getRouterLinkProps({ + href, + onClick: () => apmLinkToServiceEntityLocator?.navigate({ serviceName: props.text }), + }) + : undefined; + + return ( + + {canViewApm && isEntityCentricExperienceSettingEnabled && routeLinkProps + ? ({ content }) => {content} + : undefined} + + ); +} diff --git a/src/plugins/discover/public/components/discover_grid/virtual_columns/logs/resource.tsx b/src/plugins/discover/public/components/discover_grid/virtual_columns/logs/resource.tsx index 1cd2ef6ecc55f..e00a84228ed0f 100644 --- a/src/plugins/discover/public/components/discover_grid/virtual_columns/logs/resource.tsx +++ b/src/plugins/discover/public/components/discover_grid/virtual_columns/logs/resource.tsx @@ -15,6 +15,7 @@ import { LogDocument } from '@kbn/discover-utils/src'; import * as constants from '../../../../../common/data_types/logs/constants'; import { getUnformattedResourceFields } from './utils/resource'; import { ChipWithPopover } from '../../../data_types/logs/popover_chip'; +import { ServiceNameChipWithPopover } from '../../../data_types/logs/service_name_chip_with_popover'; const AgentIcon = dynamic(() => import('@kbn/custom-icons/src/components/agent_icon')); @@ -23,7 +24,7 @@ export const Resource = ({ row }: DataGridCellValueElementProps) => { return (
{(resourceDoc[constants.SERVICE_NAME_FIELD] as string) && ( - { - const mountComponent = (innerContent: JSX.Element) => { - return mountWithIntl( - - - {innerContent} - - - ); - }; - - it('should start successfully', () => { - const buttonSubjToTestStart = 'discoverTourButtonTestStart'; - const InnerComponent = () => { - const tourContext = useDiscoverTourContext(); - - return ( - - {'Start the tour'} - - ); - }; - - const component = mountComponent(); - // all steps are hidden by default - expect(component.find(EuiTourStep)).toHaveLength(0); - - // one step should become visible after the tour is triggered - component.find(`button[data-test-subj="${buttonSubjToTestStart}"]`).at(0).simulate('click'); - - expect(component.find(EuiTourStep)).toHaveLength(5); - expect( - component.find({ anchor: DISCOVER_TOUR_STEP_ANCHORS.addFields, isStepOpen: true }) - ).toHaveLength(1); - expect( - component.find({ anchor: DISCOVER_TOUR_STEP_ANCHORS.reorderColumns, isStepOpen: false }) - ).toHaveLength(1); - expect( - component.find({ anchor: DISCOVER_TOUR_STEP_ANCHORS.sort, isStepOpen: false }) - ).toHaveLength(1); - expect( - component.find({ anchor: DISCOVER_TOUR_STEP_ANCHORS.changeRowHeight, isStepOpen: false }) - ).toHaveLength(1); - expect( - component.find({ anchor: DISCOVER_TOUR_STEP_ANCHORS.expandDocument, isStepOpen: false }) - ).toHaveLength(1); - }); -}); diff --git a/src/plugins/discover/public/components/discover_tour/discover_tour_anchors.tsx b/src/plugins/discover/public/components/discover_tour/discover_tour_anchors.tsx deleted file mode 100644 index 58181272d5e0c..0000000000000 --- a/src/plugins/discover/public/components/discover_tour/discover_tour_anchors.tsx +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { htmlIdGenerator } from '@elastic/eui'; - -export const DISCOVER_TOUR_STEP_ANCHOR_IDS = { - addFields: htmlIdGenerator('dsc-tour-step-add-fields')(), - expandDocument: htmlIdGenerator('dsc-tour-step-expand')(), -}; - -export const DISCOVER_TOUR_STEP_ANCHORS = { - addFields: `[data-test-subj="fieldListGroupedAvailableFields-count"], #${DISCOVER_TOUR_STEP_ANCHOR_IDS.addFields}`, - reorderColumns: '[data-test-subj="dataGridColumnSelectorButton"]', - sort: '[data-test-subj="dataGridColumnSortingButton"]', - changeRowHeight: '[data-test-subj="dataGridDisplaySelectorButton"]', - expandDocument: `#${DISCOVER_TOUR_STEP_ANCHOR_IDS.expandDocument}`, -}; diff --git a/src/plugins/discover/public/components/discover_tour/discover_tour_context.ts b/src/plugins/discover/public/components/discover_tour/discover_tour_context.ts deleted file mode 100644 index 073e51389e704..0000000000000 --- a/src/plugins/discover/public/components/discover_tour/discover_tour_context.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { createContext, useContext } from 'react'; - -export interface DiscoverTourContextProps { - onStartTour: () => void; - onNextTourStep: () => void; - onFinishTour: () => void; -} - -export const DiscoverTourContext = createContext({ - onStartTour: () => {}, - onNextTourStep: () => {}, - onFinishTour: () => {}, -}); - -export const useDiscoverTourContext = () => { - return useContext(DiscoverTourContext); -}; diff --git a/src/plugins/discover/public/components/discover_tour/discover_tour_provider.tsx b/src/plugins/discover/public/components/discover_tour/discover_tour_provider.tsx deleted file mode 100644 index 846aa56c4481b..0000000000000 --- a/src/plugins/discover/public/components/discover_tour/discover_tour_provider.tsx +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React, { ReactElement, useCallback, useMemo } from 'react'; -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; -import { - useEuiTour, - EuiTourState, - EuiTourStep, - EuiTourStepProps, - EuiFlexGroup, - EuiFlexItem, - EuiButtonEmpty, - EuiButton, - EuiButtonProps, - EuiImage, - EuiSpacer, - EuiI18n, - EuiIcon, - EuiText, -} from '@elastic/eui'; -import { PLUGIN_ID } from '../../../common'; -import { useDiscoverServices } from '../../hooks/use_discover_services'; -import { DiscoverTourContext, DiscoverTourContextProps } from './discover_tour_context'; -import { DISCOVER_TOUR_STEP_ANCHORS } from './discover_tour_anchors'; -import { useIsEsqlMode } from '../../application/main/hooks/use_is_esql_mode'; - -const MAX_WIDTH = 350; - -interface TourStepDefinition { - anchor: EuiTourStepProps['anchor']; - title: EuiTourStepProps['title']; - content: EuiTourStepProps['content']; - imageName: string; - imageAltText: string; -} - -const ADD_FIELDS_STEP = { - anchor: DISCOVER_TOUR_STEP_ANCHORS.addFields, - title: i18n.translate('discover.dscTour.stepAddFields.title', { - defaultMessage: 'Add fields to the table', - }), - content: ( - , - }} - /> - ), - imageName: 'add_fields.gif', - imageAltText: i18n.translate('discover.dscTour.stepAddFields.imageAltText', { - defaultMessage: - 'In the Available fields list, click the plus icon to toggle a field into the document table.', - }), -}; - -const ORDER_TABLE_COLUMNS_STEP = { - anchor: DISCOVER_TOUR_STEP_ANCHORS.reorderColumns, - title: i18n.translate('discover.dscTour.stepReorderColumns.title', { - defaultMessage: 'Order the table columns', - }), - content: ( - - ), - imageName: 'reorder_columns.gif', - imageAltText: i18n.translate('discover.dscTour.stepReorderColumns.imageAltText', { - defaultMessage: 'Use the Columns popover to drag the columns to the order you prefer.', - }), -}; - -const CHANGE_ROW_HEIGHT_STEP = { - anchor: DISCOVER_TOUR_STEP_ANCHORS.changeRowHeight, - title: i18n.translate('discover.dscTour.stepChangeRowHeight.title', { - defaultMessage: 'Change the row height', - }), - content: ( - - ), - imageName: 'rows_per_line.gif', - imageAltText: i18n.translate('discover.dscTour.stepChangeRowHeight.imageAltText', { - defaultMessage: 'Click the display options icon to adjust the row height to fit the contents.', - }), -}; - -const tourStepDefinitions: TourStepDefinition[] = [ - ADD_FIELDS_STEP, - ORDER_TABLE_COLUMNS_STEP, - { - anchor: DISCOVER_TOUR_STEP_ANCHORS.sort, - title: i18n.translate('discover.dscTour.stepSort.title', { - defaultMessage: 'Sort on one or more fields', - }), - content: ( - - ), - imageName: 'sort.gif', - imageAltText: i18n.translate('discover.dscTour.stepSort.imageAltText', { - defaultMessage: - 'Click a column header and select the desired sort order. Adjust a multi-field sort using the fields sorted popover.', - }), - }, - CHANGE_ROW_HEIGHT_STEP, - { - anchor: DISCOVER_TOUR_STEP_ANCHORS.expandDocument, - title: i18n.translate('discover.dscTour.stepExpand.title', { - defaultMessage: 'Expand documents', - }), - content: ( - - ), - }} - /> - ), - imageName: 'expand.gif', - imageAltText: i18n.translate('discover.dscTour.stepExpand.imageAltText', { - defaultMessage: - 'Click the expand icon to inspect and filter the fields in the document and view the document in context.', - }), - }, -]; - -const FIRST_STEP = 1; - -const prepareTourSteps = ( - stepDefinitions: TourStepDefinition[], - getAssetPath: (imageName: string) => string -): EuiTourStepProps[] => - stepDefinitions.map((stepDefinition, index) => ({ - step: index + 1, - anchor: stepDefinition.anchor, - title: stepDefinition.title, - maxWidth: MAX_WIDTH, - content: ( - <> - -

{stepDefinition.content}

- - {stepDefinition.imageName && ( - <> - - - - )} - - ), - })) as EuiTourStepProps[]; - -const findNextAvailableStep = ( - steps: EuiTourStepProps[], - currentTourStep: number -): number | null => { - const nextStep = steps.find( - (step) => - step.step > currentTourStep && - typeof step.anchor === 'string' && - document.querySelector(step.anchor) - ); - - return nextStep?.step ?? null; -}; - -const tourConfig: EuiTourState = { - currentTourStep: FIRST_STEP, - isTourActive: false, - tourPopoverWidth: MAX_WIDTH, - tourSubtitle: '', -}; - -export const DiscoverTourProvider = ({ children }: { children: ReactElement }) => { - const services = useDiscoverServices(); - const isEsqlMode = useIsEsqlMode(); - const prependToBasePath = services.core.http.basePath.prepend; - const getAssetPath = useCallback( - (imageName: string) => { - return prependToBasePath(`/plugins/${PLUGIN_ID}/assets/discover_tour/${imageName}`); - }, - [prependToBasePath] - ); - const tourSteps = useMemo( - () => - isEsqlMode - ? prepareTourSteps( - [ADD_FIELDS_STEP, ORDER_TABLE_COLUMNS_STEP, CHANGE_ROW_HEIGHT_STEP], - getAssetPath - ) - : prepareTourSteps(tourStepDefinitions, getAssetPath), - [getAssetPath, isEsqlMode] - ); - const [steps, actions, reducerState] = useEuiTour(tourSteps, tourConfig); - const currentTourStep = reducerState.currentTourStep; - const isTourActive = reducerState.isTourActive; - - const onStartTour = useCallback(() => { - actions.resetTour(); - actions.goToStep(FIRST_STEP, true); - }, [actions]); - - const onNextTourStep = useCallback(() => { - const nextAvailableStep = findNextAvailableStep(steps, currentTourStep); - if (nextAvailableStep) { - actions.goToStep(nextAvailableStep); - } else { - actions.finishTour(); - } - }, [actions, steps, currentTourStep]); - - const onFinishTour = useCallback(() => { - actions.finishTour(); - }, [actions]); - - const contextValue: DiscoverTourContextProps = useMemo( - () => ({ - onStartTour, - onNextTourStep, - onFinishTour, - }), - [onStartTour, onNextTourStep, onFinishTour] - ); - - return ( - - {isTourActive && - steps.map((step) => ( - - } - /> - ))} - {children} - - ); -}; - -export const DiscoverTourStepFooterAction: React.FC<{ - isLastStep: boolean; - onNextTourStep: DiscoverTourContextProps['onNextTourStep']; - onFinishTour: DiscoverTourContextProps['onFinishTour']; -}> = ({ isLastStep, onNextTourStep, onFinishTour }) => { - const actionButtonProps: Partial = { - size: 's', - color: 'success', - }; - - return ( - - {!isLastStep && ( - - - {EuiI18n({ token: 'core.euiTourFooter.skipTour', default: 'Skip tour' })} - - - )} - - {isLastStep ? ( - - {EuiI18n({ token: 'core.euiTourFooter.endTour', default: 'End tour' })} - - ) : ( - - {EuiI18n({ token: 'core.euiTourFooter.nextStep', default: 'Next' })} - - )} - - - ); -}; diff --git a/src/plugins/discover/public/components/discover_tour/index.ts b/src/plugins/discover/public/components/discover_tour/index.ts deleted file mode 100644 index 6adcad7178ba0..0000000000000 --- a/src/plugins/discover/public/components/discover_tour/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -export { DiscoverTourProvider } from './discover_tour_provider'; -export { useDiscoverTourContext } from './discover_tour_context'; -export { DISCOVER_TOUR_STEP_ANCHOR_IDS, DISCOVER_TOUR_STEP_ANCHORS } from './discover_tour_anchors'; diff --git a/src/plugins/discover/tsconfig.json b/src/plugins/discover/tsconfig.json index 51e797b179952..fef4b38d59cd5 100644 --- a/src/plugins/discover/tsconfig.json +++ b/src/plugins/discover/tsconfig.json @@ -96,6 +96,8 @@ "@kbn/observability-ai-assistant-plugin", "@kbn/fields-metadata-plugin", "@kbn/security-solution-common", + "@kbn/router-utils", + "@kbn/management-settings-ids", "@kbn/logs-data-access-plugin" ], "exclude": [ diff --git a/src/plugins/navigation/public/plugin.test.ts b/src/plugins/navigation/public/plugin.test.ts index 02f23d16de209..a88b51abba665 100644 --- a/src/plugins/navigation/public/plugin.test.ts +++ b/src/plugins/navigation/public/plugin.test.ts @@ -97,11 +97,11 @@ describe('Navigation Plugin', () => { describe('addSolutionNavigation()', () => { it('should update the solution navigation definitions', async () => { - const { plugin, coreStart, unifiedSearch, cloud } = setup(); + const { plugin, coreStart, unifiedSearch, spaces } = setup(); const { addSolutionNavigation } = plugin.start(coreStart, { unifiedSearch, - cloud, + spaces, }); await new Promise((resolve) => setTimeout(resolve)); @@ -180,13 +180,29 @@ describe('Navigation Plugin', () => { }); describe('isSolutionNavEnabled$', () => { - // This test will need to be changed when we remove the feature flag - it('should be off by default', async () => { - const { plugin, coreStart, unifiedSearch, cloud } = setup(); + it('should be off if spaces plugin not available', async () => { + const { plugin, coreStart, unifiedSearch } = setup(); const { isSolutionNavEnabled$ } = plugin.start(coreStart, { unifiedSearch, - cloud, + }); + await new Promise((resolve) => setTimeout(resolve)); + + const isEnabled = await firstValueFrom(isSolutionNavEnabled$); + expect(isEnabled).toBe(false); + }); + + it('should be off if spaces plugin `isSolutionViewEnabled` = false', async () => { + const { plugin, coreStart, unifiedSearch, spaces } = setup(); + spaces.getActiveSpace$ = jest + .fn() + .mockReturnValue(of({ solution: 'es' } as Pick)); + + spaces.isSolutionViewEnabled = false; + + const { isSolutionNavEnabled$ } = plugin.start(coreStart, { + unifiedSearch, + spaces, }); await new Promise((resolve) => setTimeout(resolve)); diff --git a/src/plugins/navigation/public/plugin.tsx b/src/plugins/navigation/public/plugin.tsx index d2aa1f4c7f557..f382b80221642 100644 --- a/src/plugins/navigation/public/plugin.tsx +++ b/src/plugins/navigation/public/plugin.tsx @@ -72,10 +72,8 @@ export class NavigationPublicPlugin const extensions = this.topNavMenuExtensionsRegistry.getAll(); const chrome = core.chrome as InternalChromeStart; const activeSpace$: Observable = spaces?.getActiveSpace$() ?? of(undefined); - const onCloud = cloud !== undefined; // The new side nav will initially only be available to cloud users const isServerless = this.initializerContext.env.packageInfo.buildFlavor === 'serverless'; - - this.isSolutionNavEnabled = onCloud && !isServerless; + this.isSolutionNavEnabled = spaces?.isSolutionViewEnabled ?? false; /* * diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview.test.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview.test.tsx index 58d53caf529bf..56c8189479306 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview.test.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview.test.tsx @@ -15,6 +15,7 @@ import { DocViewRenderProps } from '@kbn/unified-doc-viewer/types'; import { buildDataTableRecord } from '@kbn/discover-utils'; import { setUnifiedDocViewerServices } from '../../plugin'; import { mockUnifiedDocViewerServices } from '../../__mocks__'; +import { merge } from 'lodash'; const DATASET_NAME = 'logs.overview'; const NAMESPACE = 'default'; @@ -103,7 +104,30 @@ const fullHit = buildDataTableRecord( dataView ); -setUnifiedDocViewerServices(mockUnifiedDocViewerServices); +const getCustomUnifedDocViewerServices = (params?: { + showApm: boolean; + entityCentricExperienceEnabled?: boolean; +}) => ({ + core: { + application: { + capabilities: { apm: { show: params?.showApm || false } }, + }, + uiSettings: { + get: () => params?.entityCentricExperienceEnabled || false, + }, + }, + share: { + url: { + locators: { + get: () => ({ getRedirectUrl: jest.fn().mockReturnValue('/apm/foo'), navigate: jest.fn() }), + }, + }, + }, +}); + +setUnifiedDocViewerServices( + merge(mockUnifiedDocViewerServices, getCustomUnifedDocViewerServices()) +); const renderLogsOverview = (props: Partial = {}) => { const { rerender: baseRerender, ...tools } = render( @@ -200,3 +224,85 @@ describe('LogsOverview', () => { }); }); }); + +describe('LogsOverview with APM links', () => { + describe('Highlights section', () => { + describe('When APM and Entity centric experience are enabled', () => { + beforeEach(() => { + setUnifiedDocViewerServices( + merge( + mockUnifiedDocViewerServices, + getCustomUnifedDocViewerServices({ + showApm: true, + entityCentricExperienceEnabled: true, + }) + ) + ); + renderLogsOverview(); + }); + it('should render service name link', () => { + expect( + screen.queryByTestId('unifiedDocViewLogsOverviewServiceNameHighlightLink') + ).toBeInTheDocument(); + }); + + it('should render trace id link', () => { + expect( + screen.queryByTestId('unifiedDocViewLogsOverviewTraceIdHighlightLink') + ).toBeInTheDocument(); + }); + }); + + describe('When APM is enabled and Entity centric experience is disabled', () => { + beforeEach(() => { + setUnifiedDocViewerServices( + merge( + mockUnifiedDocViewerServices, + getCustomUnifedDocViewerServices({ + showApm: true, + entityCentricExperienceEnabled: false, + }) + ) + ); + renderLogsOverview(); + }); + it('should not render service name link', () => { + expect( + screen.queryByTestId('unifiedDocViewLogsOverviewServiceNameHighlightLink') + ).not.toBeInTheDocument(); + }); + + it('should render trace id link', () => { + expect( + screen.queryByTestId('unifiedDocViewLogsOverviewTraceIdHighlightLink') + ).toBeInTheDocument(); + }); + }); + + describe('When APM is disabled and Entity centric experience is enabled', () => { + beforeEach(() => { + setUnifiedDocViewerServices( + merge( + mockUnifiedDocViewerServices, + getCustomUnifedDocViewerServices({ + showApm: false, + entityCentricExperienceEnabled: true, + }) + ) + ); + renderLogsOverview(); + }); + it('should not render service name link', () => { + expect( + screen.queryByTestId('unifiedDocViewLogsOverviewServiceNameHighlightLink') + ).not.toBeInTheDocument(); + }); + + it('should not render trace id link', () => { + expect( + screen.queryByTestId('unifiedDocViewLogsOverviewTraceIdHighlightLink') + ).not.toBeInTheDocument(); + }); + }); + }); +}); diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx index aef9289fcaf04..583e3b5c03fa1 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx @@ -15,6 +15,8 @@ import { DataTableRecord, LogDocumentOverview, fieldConstants } from '@kbn/disco import { HighlightField } from './sub_components/highlight_field'; import { HighlightSection } from './sub_components/highlight_section'; import { getUnifiedDocViewerServices } from '../../plugin'; +import { ServiceNameHighlightField } from './sub_components/service_name_highlight_field'; +import { TraceIdHighlightField } from './sub_components/trace_id_highlight_field'; export function LogsOverviewHighlights({ formattedDoc, @@ -65,7 +67,7 @@ export function LogsOverviewHighlights({ data-test-subj="unifiedDocViewLogsOverviewHighlightSectionServiceInfra" > {shouldRenderHighlight(fieldConstants.SERVICE_NAME_FIELD) && ( - )} {shouldRenderHighlight(fieldConstants.TRACE_ID_FIELD) && ( - import('./highlight_field_description')); -interface HighlightFieldProps { +export interface HighlightFieldProps { field: string; fieldMetadata?: PartialFieldMetadataPlain; formattedValue?: string; @@ -25,6 +25,7 @@ interface HighlightFieldProps { label: string; useBadge?: boolean; value?: unknown; + children?: (props: { content: React.ReactNode }) => React.ReactNode | React.ReactNode; } export function HighlightField({ @@ -35,6 +36,7 @@ export function HighlightField({ label, useBadge = false, value, + children, ...props }: HighlightFieldProps) { const hasFieldDescription = !!fieldMetadata?.short; @@ -59,13 +61,10 @@ export function HighlightField({ {formattedValue} + ) : typeof children === 'function' ? ( + children({ content: }) ) : ( - + )} @@ -73,6 +72,15 @@ export function HighlightField({ ) : null; } +const FormattedValue = ({ value }: { value: string }) => ( + +); + const fieldNameStyle = css` color: ${euiThemeVars.euiColorDarkShade}; `; diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/service_name_highlight_field.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/service_name_highlight_field.tsx new file mode 100644 index 0000000000000..351f45b9a871c --- /dev/null +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/service_name_highlight_field.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { EuiLink } from '@elastic/eui'; +import { getRouterLinkProps } from '@kbn/router-utils'; +import { OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE } from '@kbn/management-settings-ids'; +import { HighlightField, HighlightFieldProps } from './highlight_field'; +import { getUnifiedDocViewerServices } from '../../../plugin'; + +const SERVICE_ENTITY_LOCATOR = 'SERVICE_ENTITY_LOCATOR'; + +export function ServiceNameHighlightField(props: HighlightFieldProps) { + const { + share: { url: urlService }, + core, + } = getUnifiedDocViewerServices(); + const canViewApm = core.application.capabilities.apm?.show || false; + + const isEntityCentricExperienceSettingEnabled = canViewApm + ? core.uiSettings.get(OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE) + : false; + + const apmLinkToServiceEntityLocator = urlService.locators.get<{ serviceName: string }>( + SERVICE_ENTITY_LOCATOR + ); + const href = apmLinkToServiceEntityLocator?.getRedirectUrl({ + serviceName: props.value as string, + }); + + const routeLinkProps = href + ? getRouterLinkProps({ + href, + onClick: () => + apmLinkToServiceEntityLocator?.navigate({ serviceName: props.value as string }), + }) + : undefined; + + return ( + + {canViewApm && isEntityCentricExperienceSettingEnabled && routeLinkProps + ? ({ content }) => ( + + {content} + + ) + : undefined} + + ); +} diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/trace_id_highlight_field.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/trace_id_highlight_field.tsx new file mode 100644 index 0000000000000..bbcdbd0e44de5 --- /dev/null +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/trace_id_highlight_field.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { EuiLink } from '@elastic/eui'; +import { getRouterLinkProps } from '@kbn/router-utils'; +import { HighlightField, HighlightFieldProps } from './highlight_field'; +import { getUnifiedDocViewerServices } from '../../../plugin'; + +const TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR = 'TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR'; + +export function TraceIdHighlightField(props: HighlightFieldProps) { + const { + share: { url: urlService }, + core, + } = getUnifiedDocViewerServices(); + const canViewApm = core.application.capabilities.apm?.show || false; + + const apmLinkToServiceEntityLocator = urlService.locators.get<{ traceId: string }>( + TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR + ); + const href = apmLinkToServiceEntityLocator?.getRedirectUrl({ + traceId: props.value as string, + }); + + const routeLinkProps = getRouterLinkProps({ + href, + onClick: () => apmLinkToServiceEntityLocator?.navigate({ traceId: props.value as string }), + }); + return ( + + {canViewApm + ? ({ content }) => ( + + {content} + + ) + : undefined} + + ); +} diff --git a/src/plugins/unified_doc_viewer/tsconfig.json b/src/plugins/unified_doc_viewer/tsconfig.json index eab6884b972ec..212fcb0335c75 100644 --- a/src/plugins/unified_doc_viewer/tsconfig.json +++ b/src/plugins/unified_doc_viewer/tsconfig.json @@ -36,7 +36,8 @@ "@kbn/share-plugin", "@kbn/router-utils", "@kbn/unified-field-list", - "@kbn/core-lifecycle-browser" + "@kbn/core-lifecycle-browser", + "@kbn/management-settings-ids" ], "exclude": [ "target/**/*", diff --git a/test/functional/apps/console/_autocomplete.ts b/test/functional/apps/console/_autocomplete.ts index 1123c93cab49c..64db13fa67ec2 100644 --- a/test/functional/apps/console/_autocomplete.ts +++ b/test/functional/apps/console/_autocomplete.ts @@ -62,8 +62,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(PageObjects.console.isAutocompleteVisible()).to.be.eql(true); }); - // FLAKY: https://github.com/elastic/kibana/issues/186501 - describe.skip('Autocomplete behavior', () => { + describe('Autocomplete behavior', () => { beforeEach(async () => { await PageObjects.console.clearEditorText(); }); @@ -86,11 +85,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.console.isAutocompleteVisible()).to.be.eql(true); for (const [i, method] of methods.entries()) { - expect(await PageObjects.console.getAutocompleteSuggestion(i)).to.be.eql(method); + expect(await PageObjects.console.getAutocompleteSuggestion(i)).to.contain(method); } await PageObjects.console.pressEscape(); - await PageObjects.console.clearEditorText(); + await PageObjects.console.clickClearInput(); } }); @@ -354,8 +353,7 @@ GET _search }); }); - // FLAKY: https://github.com/elastic/kibana/issues/186935 - describe.skip('index fields autocomplete', () => { + describe('index fields autocomplete', () => { const indexName = `index_field_test-${Date.now()}-${Math.random()}`; before(async () => { diff --git a/test/functional/apps/console/_comments.ts b/test/functional/apps/console/_comments.ts index 0e004151d0149..abff76b20c484 100644 --- a/test/functional/apps/console/_comments.ts +++ b/test/functional/apps/console/_comments.ts @@ -15,8 +15,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const log = getService('log'); const PageObjects = getPageObjects(['common', 'console', 'header']); - // flaky - describe.skip('console app', function testComments() { + describe('console app', function testComments() { this.tags('includeFirefox'); before(async () => { log.debug('navigateTo console'); diff --git a/test/functional/apps/console/_console.ts b/test/functional/apps/console/_console.ts index ffba2a7ac41c4..6b6b192373ea6 100644 --- a/test/functional/apps/console/_console.ts +++ b/test/functional/apps/console/_console.ts @@ -53,8 +53,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.console.isOutputPanelEmptyStateVisible()).to.be(false); }); - // the resizer doesn't work the same as in ace https://github.com/elastic/kibana/issues/184352 - it.skip('should resize the editor', async () => { + it('should resize the editor', async () => { const editor = await PageObjects.console.getEditor(); await browser.setWindowSize(1300, 1100); const initialSize = await editor.getSize(); @@ -149,7 +148,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.console.clearEditorText(); }); it('it should send successful request to Kibana API', async () => { - const expectedResponseContains = 'default space'; + const expectedResponseContains = '"name": "Default"'; await PageObjects.console.enterText('GET kbn:/api/spaces/space'); await PageObjects.console.clickPlay(); await retry.try(async () => { @@ -160,8 +159,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // Status badge is not yet implented in phase 2 - describe.skip('with query params', () => { + describe('with query params', () => { it('should issue a successful request', async () => { await PageObjects.console.clearEditorText(); await PageObjects.console.enterText('GET _cat/aliases?format=json&v=true&pretty=true'); @@ -203,8 +201,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await retry.try(async () => { const response = await PageObjects.console.getOutputText(); log.debug(response); - expect(response).to.contain('# 2: PUT test-index 200'); - expect(response).to.contain('# 3: DELETE test-index 200'); + expect(response).to.contain('# 2: PUT test-index [200 OK]'); + expect(response).to.contain('# 3: DELETE test-index [200 OK]'); }); }); diff --git a/test/functional/apps/console/_misc_console_behavior.ts b/test/functional/apps/console/_misc_console_behavior.ts index c0f4ffdd0654a..68bcc6e7558cf 100644 --- a/test/functional/apps/console/_misc_console_behavior.ts +++ b/test/functional/apps/console/_misc_console_behavior.ts @@ -25,6 +25,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.common.navigateToApp('console'); // Ensure that the text area can be interacted with await PageObjects.console.skipTourIfExists(); + + await PageObjects.console.openConfig(); + await PageObjects.console.toggleKeyboardShortcuts(true); + await PageObjects.console.openConsole(); }); beforeEach(async () => { @@ -131,13 +135,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // Settings not yet implemented - it.skip('can toggle keyboard shortcuts', async () => { + it('can toggle keyboard shortcuts', async () => { // Enter a sample command await PageObjects.console.enterText('GET _search'); // Disable keyboard shorcuts + await PageObjects.console.openConfig(); await PageObjects.console.toggleKeyboardShortcuts(false); + await PageObjects.console.openConsole(); // Upon clicking ctrl enter a newline character should be added to the editor await PageObjects.console.pressCtrlEnter(); @@ -145,11 +150,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.console.isOutputPanelEmptyStateVisible()).to.be(true); // Restore setting + await PageObjects.console.openConfig(); await PageObjects.console.toggleKeyboardShortcuts(true); + await PageObjects.console.openConsole(); }); describe('customizable font size', () => { - // flaky it('should allow the font size to be customized', async () => { await PageObjects.console.openConfig(); await PageObjects.console.setFontSizeSetting(20); diff --git a/test/functional/apps/console/_output_panel.ts b/test/functional/apps/console/_output_panel.ts index 8aa548ce4b139..49c41ae7a6ccc 100644 --- a/test/functional/apps/console/_output_panel.ts +++ b/test/functional/apps/console/_output_panel.ts @@ -65,7 +65,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await sendMultipleRequests(['\n GET /_search?pretty', '\n GET /_search?pretty']); const response = await PageObjects.console.getOutputText(); - expect(response).to.contain('# 2: GET /_search?pretty'); + expect(response).to.contain('# 2: GET /_search?pretty [200 OK]'); }); it('should clear the console output', async () => { diff --git a/test/functional/apps/console/_variables.ts b/test/functional/apps/console/_variables.ts index 8b5cc0f078e3f..a5eef8cccd15d 100644 --- a/test/functional/apps/console/_variables.ts +++ b/test/functional/apps/console/_variables.ts @@ -15,7 +15,8 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { const log = getService('log'); const PageObjects = getPageObjects(['common', 'console', 'header']); - describe('Console variables', function testConsoleVariables() { + // Failing: See https://github.com/elastic/kibana/issues/157776 + describe.skip('Console variables', function testConsoleVariables() { this.tags('includeFirefox'); before(async () => { @@ -65,10 +66,15 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { }); }); + // Flaky: https://github.com/elastic/kibana/issues/157776 + // Beware that this test will pass locally and in flaky test runner, but it + // will fail after merged. describe.skip('with variables in request body', () => { - // bug in monaco https://github.com/elastic/kibana/issues/185999 - it.skip('should send a successful request', async () => { + it('should send a successful request', async () => { + await PageObjects.console.openConfig(); await PageObjects.console.addNewVariable({ name: 'query1', value: '{"match_all": {}}' }); + await PageObjects.console.openConsole(); + await PageObjects.console.clickClearInput(); await PageObjects.console.enterText('\n GET _search\n'); await PageObjects.console.enterText(`{\n\t"query": "\${query1}"`); await PageObjects.console.clickPlay(); diff --git a/test/functional/apps/dashboard/group4/dashboard_empty.ts b/test/functional/apps/dashboard/group4/dashboard_empty.ts index 1d38b777cd691..574181b614d01 100644 --- a/test/functional/apps/dashboard/group4/dashboard_empty.ts +++ b/test/functional/apps/dashboard/group4/dashboard_empty.ts @@ -19,7 +19,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const dataViews = getService('dataViews'); const { common, dashboard, header } = getPageObjects(['common', 'dashboard', 'header']); - describe('dashboard empty state', () => { + // Failing: See https://github.com/elastic/kibana/issues/165745 + describe.skip('dashboard empty state', () => { const kbnDirectory = 'test/functional/fixtures/kbn_archiver/dashboard/current/kibana'; before(async function () { diff --git a/test/functional/apps/discover/group6/_view_mode_toggle.ts b/test/functional/apps/discover/group6/_view_mode_toggle.ts index fcb6f6e9a5eb2..aec1cb6f459ab 100644 --- a/test/functional/apps/discover/group6/_view_mode_toggle.ts +++ b/test/functional/apps/discover/group6/_view_mode_toggle.ts @@ -68,10 +68,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await documentsTab.getAttribute('aria-selected')).to.be('true'); }); - it('should show Document Explorer info callout', async () => { - await testSubjects.existOrFail( - useLegacyTable ? 'dscDocumentExplorerLegacyCallout' : 'dscDocumentExplorerTourCallout' - ); + it('should show legacy Document Explorer info callout', async () => { + if (useLegacyTable) { + await testSubjects.existOrFail('dscDocumentExplorerLegacyCallout'); + } else { + await testSubjects.missingOrFail('dscDocumentExplorerLegacyCallout'); + } }); it('should show an error callout', async () => { diff --git a/test/functional/apps/discover/group8/_hide_announcements.ts b/test/functional/apps/discover/group8/_hide_announcements.ts index aa1943b687c3f..9dc11f9eefcac 100644 --- a/test/functional/apps/discover/group8/_hide_announcements.ts +++ b/test/functional/apps/discover/group8/_hide_announcements.ts @@ -23,7 +23,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); - await kibanaServer.uiSettings.replace({ defaultIndex: 'logstash-*' }); + await kibanaServer.uiSettings.replace({ + defaultIndex: 'logstash-*', + 'doc_table:legacy': true, + }); await common.navigateToApp('discover'); await timePicker.setDefaultAbsoluteRange(); }); @@ -32,17 +35,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await kibanaServer.uiSettings.replace({}); }); - it('should display take tour button', async function () { + it('should display callout', async function () { await discover.selectIndexPattern('logstash-*'); - const tourButtonExists = await testSubjects.exists('discoverTakeTourButton'); - expect(tourButtonExists).to.be(true); + const calloutExists = await testSubjects.exists('dscDocumentExplorerLegacyCallout'); + expect(calloutExists).to.be(true); }); - it('should not display take tour button', async function () { + it('should not display callout', async function () { await kibanaServer.uiSettings.update({ hideAnnouncements: true }); await browser.refresh(); - const tourButtonExists = await testSubjects.exists('discoverTakeTourButton'); - expect(tourButtonExists).to.be(false); + const calloutExists = await testSubjects.exists('dscDocumentExplorerLegacyCallout'); + expect(calloutExists).to.be(false); }); }); } diff --git a/test/plugin_functional/test_suites/data_plugin/session.ts b/test/plugin_functional/test_suites/data_plugin/session.ts index d69a7529a19ad..c44e606aa2d04 100644 --- a/test/plugin_functional/test_suites/data_plugin/session.ts +++ b/test/plugin_functional/test_suites/data_plugin/session.ts @@ -32,6 +32,7 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide return sessionIds.split(','); }; + // Failing: See https://github.com/elastic/kibana/issues/192510 // Failing: See https://github.com/elastic/kibana/issues/192510 describe.skip('Session management', function describeSessionManagementTests() { describe('Discover', () => { diff --git a/x-pack/performance/journeys_e2e/many_fields_discover.ts b/x-pack/performance/journeys_e2e/many_fields_discover.ts index 2a801dea4478f..196445f7fff71 100644 --- a/x-pack/performance/journeys_e2e/many_fields_discover.ts +++ b/x-pack/performance/journeys_e2e/many_fields_discover.ts @@ -24,7 +24,7 @@ export const journey = new Journey({ }) .step('Expand the first document', async ({ page }) => { const expandButtons = page.locator(subj('docTableExpandToggleColumn')); - await expandButtons.first().click(); + await expandButtons.last().click(); await page.waitForSelector(subj('docTableRowAction')); await page.click(subj('docTableRowAction')); await page.waitForSelector(subj('globalLoadingIndicator-hidden')); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts new file mode 100644 index 0000000000000..b9119143ec37e --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; +import { BUILT_IN_ID_PREFIX } from './constants'; + +export const builtInContainersFromEcsEntityDefinition: EntityDefinition = + entityDefinitionSchema.parse({ + id: `${BUILT_IN_ID_PREFIX}containers_from_ecs_data`, + managed: true, + version: '1.0.0', + name: 'Containers from ECS data', + description: + 'This definition extracts container entities from common data streams by looking for the ECS field container.id', + type: 'container', + indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], + identityFields: ['container.id'], + displayNameTemplate: '{{container.id}}', + history: { + timestampField: '@timestamp', + interval: '5m', + settings: { + frequency: '5m', + }, + }, + metadata: [ + { + source: '_index', + destination: 'source_index', + }, + { + source: 'data_stream.type', + destination: 'source_data_stream.type', + }, + { + source: 'data_stream.dataset', + destination: 'source_data_stream.dataset', + }, + 'container.name', + 'container.image.name', + 'container.image.tag', + 'container.runtime', + 'host.name', + 'host.ip', + 'host.mac', + 'host.architecture', + 'host.os.family', + 'host.os.kernel', + 'host.os.name', + 'host.os.platform', + 'host.os.type', + 'host.os.version', + 'cloud.provider', + 'cloud.region', + 'cloud.availability_zone', + 'cloud.instance.id', + 'cloud.instance.name', + 'cloud.machine.type', + 'cloud.service.name', + 'agent.name', + 'agent.type', + 'agent.ephemeral_id', + ], + metrics: [ + { + name: 'log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: * OR error.log.level: *', + }, + ], + }, + { + name: 'error_log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")', + }, + ], + }, + { + name: 'cpu_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.cpu.total.pct', + }, + ], + }, + { + name: 'memory_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.memory.usage.pct', + }, + ], + }, + { + name: 'network_in_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.network.in.bytes', + }, + ], + }, + { + name: 'network_out_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.network.out.bytes', + }, + ], + }, + { + name: 'disk_read_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.diskio.read.ops', + }, + ], + }, + { + name: 'disk_write_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'docker.diskio.write.ops', + }, + ], + }, + ], + }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts new file mode 100644 index 0000000000000..5fead32f5c0e8 --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; +import { BUILT_IN_ID_PREFIX } from './constants'; + +export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ + id: `${BUILT_IN_ID_PREFIX}hosts_from_ecs_data`, + managed: true, + version: '1.0.0', + name: 'Hosts from ECS data', + description: + 'This definition extracts host entities from common data streams by looking for the ECS field host.name', + type: 'host', + indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'], + identityFields: ['host.name'], + displayNameTemplate: '{{host.name}}', + history: { + timestampField: '@timestamp', + interval: '5m', + settings: { + frequency: '5m', + }, + }, + metadata: [ + { + source: '_index', + destination: 'source_index', + }, + { + source: 'data_stream.type', + destination: 'source_data_stream.type', + }, + { + source: 'data_stream.dataset', + destination: 'source_data_stream.dataset', + }, + 'host.hostname', + 'host.ip', + 'host.mac', + 'host.architecture', + 'host.containerized', + 'host.os.platform', + 'host.os.name', + 'host.os.type', + 'host.os.codename', + 'host.os.family', + 'host.os.kernel', + 'host.os.version', + 'cloud.provider', + 'cloud.region', + 'cloud.availability_zone', + 'cloud.instance.id', + 'cloud.instance.name', + 'cloud.service.name', + 'cloud.machine.type', + 'cloud.account.id', + 'cloud.project.id', + 'agent.id', + 'agent.name', + 'agent.type', + 'agent.version', + ], + metrics: [ + { + name: 'log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: * OR error.log.level: *', + }, + ], + }, + { + name: 'error_log_rate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")', + }, + ], + }, + { + name: 'cpu_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'system.cpu.total.norm.pct', + }, + ], + }, + { + name: 'normalized_load_avg', + equation: 'A / B', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'system.load.1', + }, + { + name: 'B', + aggregation: 'max', + field: 'system.load.cores', + }, + ], + }, + { + name: 'memory_usage_avg', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + field: 'system.memory.actual.used.pct', + }, + ], + }, + { + name: 'memory_free_avg', + equation: 'A - B', + metrics: [ + { + name: 'A', + aggregation: 'max', + field: 'system.memory.total', + }, + { + name: 'B', + aggregation: 'avg', + field: 'system.memory.actual.used.bytes', + }, + ], + }, + { + name: 'disk_usage_max', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'max', + field: 'system.filesystem.used.pct', + }, + ], + }, + { + name: 'rx_avg', + equation: 'A * 8', + metrics: [ + { + name: 'A', + aggregation: 'sum', + field: 'host.network.ingress.bytes', + }, + ], + }, + { + name: 'tx_avg', + equation: 'A * 8', + metrics: [ + { + name: 'A', + aggregation: 'sum', + field: 'host.network.egress.bytes', + }, + ], + }, + ], +}); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts index d091e21f446d2..6c0d4c5995c63 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts @@ -6,8 +6,14 @@ */ import { EntityDefinition } from '@kbn/entities-schema'; -import { builtInServicesFromLogsEntityDefinition } from './services'; +import { builtInServicesFromEcsEntityDefinition } from './services_from_ecs_data'; +import { builtInHostsFromEcsEntityDefinition } from './hosts_from_ecs_data'; +import { builtInContainersFromEcsEntityDefinition } from './containers_from_ecs_data'; export { BUILT_IN_ID_PREFIX } from './constants'; -export const builtInDefinitions: EntityDefinition[] = [builtInServicesFromLogsEntityDefinition]; +export const builtInDefinitions: EntityDefinition[] = [ + builtInServicesFromEcsEntityDefinition, + builtInHostsFromEcsEntityDefinition, + builtInContainersFromEcsEntityDefinition, +]; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts similarity index 98% rename from x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts rename to x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index aa1d86ee25adf..96667fb4d0af4 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -18,7 +18,7 @@ const serviceTransactionFilter = (additionalFilters: string[] = []) => { return [...baseFilters, ...additionalFilters].join(' AND '); }; -export const builtInServicesFromLogsEntityDefinition: EntityDefinition = +export const builtInServicesFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ version: '1.0.3', id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`, diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index e639b364343d9..5fa4085c5106f 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -71,7 +71,7 @@ export interface FullAgentPolicyInputStream { id: string; data_stream: { dataset: string; - type: string; + type?: string; }; [key: string]: any; } diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 1e456f3e4d21d..354834d2571dc 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -90,6 +90,7 @@ export interface NewPackagePolicy { privileges?: { cluster?: string[]; }; + [key: string]: any; }; overrides?: { inputs?: { [key: string]: any } } | null; } diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_standalone.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_standalone.tsx index 1b0e791fbfd8c..0b9a6f6e85830 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_standalone.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/components/page_steps/install_agent/install_agent_standalone.tsx @@ -35,7 +35,8 @@ export const InstallElasticAgentStandalonePageStep: React.FC setPolicyCopied(true), }), diff --git a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/hooks.tsx b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/hooks.tsx index 50de1560b82b8..e6ecfbdfbd944 100644 --- a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/hooks.tsx +++ b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/hooks.tsx @@ -210,12 +210,15 @@ export function useGetCreateApiKey() { const core = useStartServices(); const [apiKey, setApiKey] = useState(undefined); + const [isLoading, setIsLoading] = useState(false); const onCreateApiKey = useCallback(async () => { try { + setIsLoading(true); const res = await sendCreateStandaloneAgentAPIKey({ name: crypto.randomBytes(16).toString('hex'), }); - const newApiKey = `${res.data?.item.id}:${res.data?.item.api_key}`; + + const newApiKey = `${res.item.id}:${res.item.api_key}`; setApiKey(newApiKey); } catch (err) { core.notifications.toasts.addError(err, { @@ -224,9 +227,11 @@ export function useGetCreateApiKey() { }), }); } + setIsLoading(false); }, [core.notifications.toasts]); return { apiKey, + isLoading, onCreateApiKey, }; } @@ -235,7 +240,7 @@ export function useFetchFullPolicy(agentPolicy: AgentPolicy | undefined, isK8s?: const core = useStartServices(); const [yaml, setYaml] = useState(''); const [fullAgentPolicy, setFullAgentPolicy] = useState(); - const { apiKey, onCreateApiKey } = useGetCreateApiKey(); + const { apiKey, isLoading: isCreatingApiKey, onCreateApiKey } = useGetCreateApiKey(); useEffect(() => { async function fetchFullPolicy() { @@ -302,6 +307,7 @@ export function useFetchFullPolicy(agentPolicy: AgentPolicy | undefined, isK8s?: yaml, onCreateApiKey, fullAgentPolicy, + isCreatingApiKey, apiKey, downloadYaml, }; diff --git a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/compute_steps.tsx b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/compute_steps.tsx index bc4e7755044e7..ea31b163fb368 100644 --- a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/compute_steps.tsx +++ b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/compute_steps.tsx @@ -54,7 +54,10 @@ export const StandaloneSteps: React.FunctionComponent = ({ isK8s, cloudSecurityIntegration, }) => { - const { yaml, onCreateApiKey, apiKey, downloadYaml } = useFetchFullPolicy(selectedPolicy, isK8s); + const { yaml, onCreateApiKey, isCreatingApiKey, apiKey, downloadYaml } = useFetchFullPolicy( + selectedPolicy, + isK8s + ); const agentVersion = useAgentVersion(); @@ -88,6 +91,7 @@ export const StandaloneSteps: React.FunctionComponent = ({ downloadYaml, apiKey, onCreateApiKey, + isCreatingApiKey, }) ); @@ -116,6 +120,7 @@ export const StandaloneSteps: React.FunctionComponent = ({ downloadYaml, apiKey, onCreateApiKey, + isCreatingApiKey, cloudSecurityIntegration, mode, setMode, diff --git a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/configure_standalone_agent_step.tsx b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/configure_standalone_agent_step.tsx index 36d6eddd8eecc..9fc74b3562ab0 100644 --- a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/configure_standalone_agent_step.tsx +++ b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/steps/configure_standalone_agent_step.tsx @@ -34,6 +34,7 @@ export const ConfigureStandaloneAgentStep = ({ downloadYaml, apiKey, onCreateApiKey, + isCreatingApiKey, isComplete, onCopy, }: { @@ -43,6 +44,7 @@ export const ConfigureStandaloneAgentStep = ({ downloadYaml: () => void; apiKey: string | undefined; onCreateApiKey: () => void; + isCreatingApiKey: boolean; isComplete?: boolean; onCopy?: () => void; }): EuiContainedStepProps => { @@ -167,7 +169,7 @@ export const ConfigureStandaloneAgentStep = ({ - + ({ + return sendRequestForRq({ method: 'post', path: CREATE_STANDALONE_AGENT_API_KEY_ROUTE, version: API_VERSIONS.internal.v1, diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index fbcf1ee80f206..05876c2bbdf26 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -9,9 +9,20 @@ import { httpServerMock, httpServiceMock } from '@kbn/core/server/mocks'; import type { KibanaRequest } from '@kbn/core/server'; import type { RouteConfig } from '@kbn/core/server'; +import type { + ListResult, + PostDeletePackagePoliciesResponse, + UpgradePackagePolicyResponse, +} from '../../../common'; + import type { FleetAuthzRouter } from '../../services/security'; import { PACKAGE_POLICY_API_ROUTES } from '../../../common/constants'; +import type { + DryRunPackagePolicy, + UpgradePackagePolicyDryRunResponse, + UpgradePackagePolicyDryRunResponseItem, +} from '../../../common/types'; import { agentPolicyService, appContextService, @@ -21,10 +32,33 @@ import { import { createAppContextStartContractMock, xpackMocks } from '../../mocks'; import type { PackagePolicyClient, FleetRequestHandlerContext } from '../..'; import type { UpdatePackagePolicyRequestSchema } from '../../types/rest_spec'; -import type { AgentPolicy, FleetRequestHandler } from '../../types'; +import { + PackagePolicyResponseSchema, + type AgentPolicy, + type FleetRequestHandler, + BulkGetPackagePoliciesResponseBodySchema, + DeletePackagePoliciesResponseBodySchema, + DeleteOnePackagePolicyResponseSchema, + UpgradePackagePoliciesResponseBodySchema, + DryRunPackagePoliciesResponseBodySchema, + OrphanedPackagePoliciesResponseSchema, + CreatePackagePolicyResponseSchema, +} from '../../types'; import type { PackagePolicy } from '../../types'; -import { getPackagePoliciesHandler } from './handlers'; +import { ListResponseSchema } from '../schema/utils'; + +import { + bulkGetPackagePoliciesHandler, + createPackagePolicyHandler, + deleteOnePackagePolicyHandler, + deletePackagePolicyHandler, + dryRunUpgradePackagePolicyHandler, + getOnePackagePolicyHandler, + getOrphanedPackagePolicies, + getPackagePoliciesHandler, + upgradePackagePolicyHandler, +} from './handlers'; import { registerRoutes } from '.'; const packagePolicyServiceMock = packagePolicyService as jest.Mocked; @@ -79,35 +113,7 @@ jest.mock( delete: jest.fn(), get: jest.fn(), getByIDs: jest.fn(), - list: jest.fn(async (_, __) => { - return { - total: 1, - perPage: 10, - page: 1, - items: [ - { - id: `123`, - name: `Package Policy 123`, - description: '', - created_at: '2022-12-19T20:43:45.879Z', - created_by: 'elastic', - updated_at: '2022-12-19T20:43:45.879Z', - updated_by: 'elastic', - policy_id: `agent-policy-id-a`, - policy_ids: [`agent-policy-id-a`], - enabled: true, - inputs: [], - namespace: 'default', - package: { - name: 'a-package', - title: 'package A', - version: '1.0.0', - }, - revision: 1, - }, - ], - }; - }), + list: jest.fn(), listIds: jest.fn(), update: jest.fn(), // @ts-ignore @@ -131,6 +137,7 @@ jest.mock('../../services/agent_policy', () => { agentPolicyService: { get: jest.fn(), update: jest.fn(), + list: jest.fn(), }, }; }); @@ -140,9 +147,18 @@ jest.mock('../../services/epm/packages', () => { ensureInstalledPackage: jest.fn(() => Promise.resolve()), getPackageInfo: jest.fn(() => Promise.resolve()), getInstallation: jest.fn(), + getInstallations: jest.fn().mockResolvedValue({ + saved_objects: [ + { + attributes: { name: 'a-package', version: '1.0.0' }, + }, + ], + }), }; }); +let testPackagePolicy: PackagePolicy; + describe('When calling package policy', () => { let routerMock: jest.Mocked; let routeHandler: FleetRequestHandler; @@ -160,6 +176,65 @@ describe('When calling package policy', () => { context = xpackMocks.createRequestHandlerContext() as unknown as FleetRequestHandlerContext; (await context.fleet).packagePolicyService.asCurrentUser as jest.Mocked; response = httpServerMock.createResponseFactory(); + testPackagePolicy = { + agents: 100, + created_at: '2022-12-19T20:43:45.879Z', + created_by: 'elastic', + description: '', + enabled: true, + id: '123', + inputs: [ + { + streams: [ + { + id: '1', + compiled_stream: {}, + enabled: true, + keep_enabled: false, + release: 'beta', + vars: { var: { type: 'text', value: 'value', frozen: false } }, + config: { config: { type: 'text', value: 'value', frozen: false } }, + data_stream: { dataset: 'apache.access', type: 'logs', elasticsearch: {} }, + }, + ], + compiled_input: '', + id: '1', + enabled: true, + type: 'logs', + policy_template: '', + keep_enabled: false, + vars: { var: { type: 'text', value: 'value', frozen: false } }, + config: { config: { type: 'text', value: 'value', frozen: false } }, + }, + ], + vars: { var: { type: 'text', value: 'value', frozen: false } }, + name: 'Package Policy 123', + namespace: 'default', + package: { + name: 'a-package', + title: 'package A', + version: '1.0.0', + experimental_data_stream_features: [{ data_stream: 'logs', features: { tsdb: true } }], + requires_root: false, + }, + policy_id: 'agent-policy-id-a', + policy_ids: ['agent-policy-id-a'], + revision: 1, + updated_at: '2022-12-19T20:43:45.879Z', + updated_by: 'elastic', + version: '1.0.0', + secret_references: [ + { + id: 'ref1', + }, + ], + spaceIds: ['space1'], + elasticsearch: { + 'index_template.mappings': { + dynamic_templates: [], + }, + }, + }; }); afterEach(() => { @@ -187,7 +262,14 @@ describe('When calling package policy', () => { }); }; - const existingPolicy = { + const existingPolicy: PackagePolicy = { + id: '1', + revision: 1, + created_at: '', + created_by: '', + updated_at: '', + updated_by: '', + policy_ids: ['2'], name: 'endpoint-1', description: 'desc', policy_id: '2', @@ -231,17 +313,10 @@ describe('When calling package policy', () => { beforeEach(() => { jest.spyOn(licenseService, 'hasAtLeast').mockClear(); packagePolicyServiceMock.update.mockImplementation((soClient, esClient, policyId, newData) => - Promise.resolve(newData as PackagePolicy) + Promise.resolve({ ...existingPolicy, ...newData } as PackagePolicy) ); packagePolicyServiceMock.get.mockResolvedValue({ - id: '1', - revision: 1, - created_at: '', - created_by: '', - updated_at: '', - updated_by: '', ...existingPolicy, - policy_ids: [existingPolicy.policy_id], inputs: [ { ...existingPolicy.inputs[0], @@ -264,6 +339,8 @@ describe('When calling package policy', () => { expect(response.ok).toHaveBeenCalledWith({ body: { item: existingPolicy }, }); + const validationResp = PackagePolicyResponseSchema.validate(existingPolicy); + expect(validationResp).toEqual(existingPolicy); }); it('should use request package policy props if provided by request', async () => { @@ -300,9 +377,13 @@ describe('When calling package policy', () => { }; const request = getUpdateKibanaRequest(newData as any); await routeHandler(context, request, response); + const responseItem = { ...existingPolicy, ...newData }; expect(response.ok).toHaveBeenCalledWith({ - body: { item: newData }, + body: { item: responseItem }, }); + + const validationResp = PackagePolicyResponseSchema.validate(responseItem); + expect(validationResp).toEqual(responseItem); }); it('should override props provided by request only', async () => { @@ -435,43 +516,43 @@ describe('When calling package policy', () => { inputs, } as any); await routeHandler(context, request, response); - expect(response.ok).toHaveBeenCalledWith({ - body: { - item: { - description: 'desc', - enabled: true, - inputs: [ + const responseItem = { + ...existingPolicy, + inputs: [ + { + type: 'input-logs', + enabled: false, + streams: [ { - type: 'input-logs', enabled: false, - streams: [ - { - enabled: false, - data_stream: { - type: 'logs', - dataset: 'test.some_logs', - }, - }, - ], + data_stream: { + type: 'logs', + dataset: 'test.some_logs', + }, }, ], - name: 'endpoint-1', - namespace: 'default', - package: { - name: 'endpoint', - title: 'Elastic Endpoint', - version: '0.5.0', - }, - vars: expect.any(Object), - policy_id: '2', }, + ], + }; + expect(response.ok).toHaveBeenCalledWith({ + body: { + item: responseItem, }, }); + + const validationResp = PackagePolicyResponseSchema.validate(responseItem); + expect(validationResp).toEqual(responseItem); }); }); describe('list api handler', () => { it('should return agent count when `withAgentCount` query param is used', async () => { + packagePolicyServiceMock.list.mockResolvedValue({ + total: 1, + perPage: 10, + page: 1, + items: [testPackagePolicy], + }); const request = httpServerMock.createKibanaRequest({ query: { withAgentCount: true, @@ -510,37 +591,334 @@ describe('When calling package policy', () => { }); await getPackagePoliciesHandler(context, request, response); + const responseBody: ListResult = { + page: 1, + perPage: 10, + total: 1, + items: [testPackagePolicy], + }; + expect(response.ok).toHaveBeenCalledWith({ + body: responseBody, + }); + + const validationResp = ListResponseSchema(PackagePolicyResponseSchema).validate(responseBody); + expect(validationResp).toEqual(responseBody); + }); + }); + + describe('bulk api handler', () => { + it('should return valid response', async () => { + const items: PackagePolicy[] = [testPackagePolicy]; + packagePolicyServiceMock.getByIDs.mockResolvedValue(items); + const request = httpServerMock.createKibanaRequest({ + query: {}, + body: { ids: ['1'] }, + }); + await bulkGetPackagePoliciesHandler(context, request, response); + expect(response.ok).toHaveBeenCalledWith({ + body: { items }, + }); + const validationResp = BulkGetPackagePoliciesResponseBodySchema.validate({ items }); + expect(validationResp).toEqual({ items }); + }); + }); + + describe('orphaned package policies api handler', () => { + it('should return valid response', async () => { + const items: PackagePolicy[] = [testPackagePolicy]; + const expectedResponse = { + items, + total: 1, + }; + packagePolicyServiceMock.list.mockResolvedValue({ + items: [testPackagePolicy], + total: 1, + page: 1, + perPage: 20, + }); + mockedAgentPolicyService.list.mockResolvedValue({ + items: [], + total: 0, + page: 1, + perPage: 20, + }); + await getOrphanedPackagePolicies(context, {} as any, response); + expect(response.ok).toHaveBeenCalledWith({ + body: expectedResponse, + }); + const validationResp = OrphanedPackagePoliciesResponseSchema.validate(expectedResponse); + expect(validationResp).toEqual(expectedResponse); + }); + }); + + describe('get api handler', () => { + it('should return valid response', async () => { + packagePolicyServiceMock.get.mockResolvedValue(testPackagePolicy); + const request = httpServerMock.createKibanaRequest({ + params: { + packagePolicyId: '1', + }, + }); + await getOnePackagePolicyHandler(context, request, response); + expect(response.ok).toHaveBeenCalledWith({ + body: { item: testPackagePolicy }, + }); + const validationResp = PackagePolicyResponseSchema.validate(testPackagePolicy); + expect(validationResp).toEqual(testPackagePolicy); + }); + + it('should return valid response simplified format', async () => { + packagePolicyServiceMock.get.mockResolvedValue(testPackagePolicy); + const request = httpServerMock.createKibanaRequest({ + params: { + packagePolicyId: '1', + }, + query: { + format: 'simplified', + }, + }); + await getOnePackagePolicyHandler(context, request, response); + const simplifiedPackagePolicy = { + ...testPackagePolicy, + inputs: { + logs: { + enabled: true, + streams: { + 'apache.access': { + enabled: true, + vars: { + var: 'value', + }, + }, + }, + vars: { + var: 'value', + }, + }, + }, + vars: { + var: 'value', + }, + }; + expect(response.ok).toHaveBeenCalledWith({ + body: { item: simplifiedPackagePolicy }, + }); + const validationResp = PackagePolicyResponseSchema.validate(simplifiedPackagePolicy); + expect(validationResp).toEqual(simplifiedPackagePolicy); + }); + }); + describe('create api handler', () => { + it('should return valid response', async () => { + packagePolicyServiceMock.get.mockResolvedValue(testPackagePolicy); + ( + (await context.fleet).packagePolicyService.asCurrentUser as jest.Mocked + ).create.mockResolvedValue(testPackagePolicy); + const request = httpServerMock.createKibanaRequest({ + body: testPackagePolicy, + }); + const expectedResponse = { item: testPackagePolicy }; + await createPackagePolicyHandler(context, request, response); expect(response.ok).toHaveBeenCalledWith({ + body: expectedResponse, + }); + const validationResp = CreatePackagePolicyResponseSchema.validate(expectedResponse); + expect(validationResp).toEqual(expectedResponse); + }); + }); + + describe('bulk delete api handler', () => { + it('should return valid response', async () => { + const responseBody: PostDeletePackagePoliciesResponse = [ + { + id: '1', + name: 'policy', + success: true, + policy_ids: ['1'], + output_id: '1', + package: { + name: 'package', + version: '1.0.0', + title: 'Package', + }, + statusCode: 409, + body: { + message: 'conflict', + }, + }, + ]; + packagePolicyServiceMock.delete.mockResolvedValue(responseBody); + const request = httpServerMock.createKibanaRequest({ body: { - page: 1, - perPage: 10, - total: 1, - items: [ - { - agents: 100, - created_at: '2022-12-19T20:43:45.879Z', - created_by: 'elastic', - description: '', - enabled: true, - id: '123', - inputs: [], - name: 'Package Policy 123', - namespace: 'default', - package: { - name: 'a-package', - title: 'package A', - version: '1.0.0', + packagePolicyIds: ['1'], + }, + }); + await deletePackagePolicyHandler(context, request, response); + expect(response.ok).toHaveBeenCalledWith({ + body: responseBody, + }); + const validationResp = DeletePackagePoliciesResponseBodySchema.validate(responseBody); + expect(validationResp).toEqual(responseBody); + }); + }); + + describe('delete api handler', () => { + it('should return valid response', async () => { + const responseBody = { + id: '1', + }; + packagePolicyServiceMock.delete.mockResolvedValue([ + { + id: '1', + name: 'policy', + success: true, + policy_ids: ['1'], + output_id: '1', + package: { + name: 'package', + version: '1.0.0', + title: 'Package', + }, + statusCode: 409, + body: { + message: 'conflict', + }, + }, + ]); + const request = httpServerMock.createKibanaRequest({ + body: { + force: false, + }, + params: { + packagePolicyId: '1', + }, + }); + await deleteOnePackagePolicyHandler(context, request, response); + expect(response.ok).toHaveBeenCalledWith({ + body: responseBody, + }); + const validationResp = DeleteOnePackagePolicyResponseSchema.validate(responseBody); + expect(validationResp).toEqual(responseBody); + }); + }); + + describe('upgrade api handler', () => { + it('should return valid response', async () => { + const responseBody: UpgradePackagePolicyResponse = [ + { + id: '1', + name: 'policy', + success: true, + statusCode: 200, + body: { + message: 'success', + }, + }, + ]; + packagePolicyServiceMock.upgrade.mockResolvedValue(responseBody); + const request = httpServerMock.createKibanaRequest({ + body: { + packagePolicyIds: ['1'], + }, + }); + await upgradePackagePolicyHandler(context, request, response); + expect(response.ok).toHaveBeenCalledWith({ + body: responseBody, + }); + const validationResp = UpgradePackagePoliciesResponseBodySchema.validate(responseBody); + expect(validationResp).toEqual(responseBody); + }); + }); + + describe('dry run upgrade api handler', () => { + it('should return valid response', async () => { + const dryRunPackagePolicy: DryRunPackagePolicy = { + description: '', + enabled: true, + id: '123', + inputs: [ + { + streams: [ + { + id: '1', + enabled: true, + keep_enabled: false, + release: 'beta', + vars: { var: { type: 'text', value: 'value', frozen: false } }, + config: { config: { type: 'text', value: 'value', frozen: false } }, + data_stream: { dataset: 'apache.access', type: 'logs', elasticsearch: {} }, }, - policy_id: 'agent-policy-id-a', - policy_ids: ['agent-policy-id-a'], + ], + id: '1', + enabled: true, + type: 'logs', + policy_template: '', + keep_enabled: false, + vars: { var: { type: 'text', value: 'value', frozen: false } }, + config: { config: { type: 'text', value: 'value', frozen: false } }, + }, + ], + vars: { var: { type: 'text', value: 'value', frozen: false } }, + name: 'Package Policy 123', + namespace: 'default', + package: { + name: 'a-package', + title: 'package A', + version: '1.0.0', + experimental_data_stream_features: [{ data_stream: 'logs', features: { tsdb: true } }], + requires_root: false, + }, + policy_id: 'agent-policy-id-a', + policy_ids: ['agent-policy-id-a'], + errors: [{ key: 'error', message: 'error' }], + missingVars: ['var'], + }; + const responseItem: UpgradePackagePolicyDryRunResponseItem = { + hasErrors: false, + name: 'policy', + statusCode: 200, + body: { + message: 'success', + }, + diff: [testPackagePolicy, dryRunPackagePolicy], + agent_diff: [ + [ + { + id: '1', + name: 'input', revision: 1, - updated_at: '2022-12-19T20:43:45.879Z', - updated_by: 'elastic', + type: 'logs', + data_stream: { namespace: 'default' }, + use_output: 'default', + package_policy_id: '1', + streams: [ + { + id: 'logfile-log.logs-d46700b2-47f8-4b1a-9153-14a717dc5edf', + data_stream: { + dataset: 'generic', + }, + paths: ['/var/tmp'], + ignore_older: '72h', + }, + ], }, ], + ], + }; + const responseBody: UpgradePackagePolicyDryRunResponse = [responseItem, responseItem]; + packagePolicyServiceMock.getUpgradeDryRunDiff.mockResolvedValueOnce(responseBody[0]); + packagePolicyServiceMock.getUpgradeDryRunDiff.mockResolvedValueOnce(responseBody[1]); + const request = httpServerMock.createKibanaRequest({ + body: { + packagePolicyIds: ['1', '2'], }, }); + await dryRunUpgradePackagePolicyHandler(context, request, response); + expect(response.ok).toHaveBeenCalledWith({ + body: responseBody, + }); + const validationResp = DryRunPackagePoliciesResponseBodySchema.validate(responseBody); + expect(validationResp).toEqual(responseBody); }); }); }); diff --git a/x-pack/plugins/fleet/server/routes/package_policy/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/index.ts index 893eb37a9b1bc..86ac38e658ee3 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/index.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { schema } from '@kbn/config-schema'; import { getRouteRequiredAuthz } from '../../services/security'; @@ -22,9 +23,21 @@ import { DryRunPackagePoliciesRequestSchema, DeleteOnePackagePolicyRequestSchema, BulkGetPackagePoliciesRequestSchema, + PackagePolicyResponseSchema, + BulkGetPackagePoliciesResponseBodySchema, + DeletePackagePoliciesResponseBodySchema, + DeleteOnePackagePolicyResponseSchema, + UpgradePackagePoliciesResponseBodySchema, + DryRunPackagePoliciesResponseBodySchema, + OrphanedPackagePoliciesResponseSchema, + CreatePackagePolicyResponseSchema, } from '../../types'; import { calculateRouteAuthz } from '../../services/security/security'; +import { genericErrorResponse, notFoundResponse } from '../schema/errors'; + +import { ListResponseSchema } from '../schema/utils'; + import { getPackagePoliciesHandler, getOnePackagePolicyHandler, @@ -48,11 +61,25 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz, getRouteRequiredAuthz('get', PACKAGE_POLICY_API_ROUTES.LIST_PATTERN) ).granted, + description: 'List package policies', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetPackagePoliciesRequestSchema }, + validate: { + request: GetPackagePoliciesRequestSchema, + response: { + 200: { + body: () => ListResponseSchema(PackagePolicyResponseSchema), + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, getPackagePoliciesHandler ); @@ -66,11 +93,28 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz, getRouteRequiredAuthz('post', PACKAGE_POLICY_API_ROUTES.BULK_GET_PATTERN) ).granted, + description: 'Bulk get package policies', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: BulkGetPackagePoliciesRequestSchema }, + validate: { + request: BulkGetPackagePoliciesRequestSchema, + response: { + 200: { + body: () => BulkGetPackagePoliciesResponseBodySchema, + }, + 400: { + body: genericErrorResponse, + }, + 404: { + body: notFoundResponse, + }, + }, + }, }, bulkGetPackagePoliciesHandler ); @@ -84,11 +128,31 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz, getRouteRequiredAuthz('get', PACKAGE_POLICY_API_ROUTES.INFO_PATTERN) ).granted, + description: 'Get package policy by ID', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetOnePackagePolicyRequestSchema }, + validate: { + request: GetOnePackagePolicyRequestSchema, + response: { + 200: { + body: () => + schema.object({ + item: PackagePolicyResponseSchema, + }), + }, + 400: { + body: genericErrorResponse, + }, + 404: { + body: notFoundResponse, + }, + }, + }, }, getOnePackagePolicyHandler ); @@ -103,20 +167,48 @@ export const registerRoutes = (router: FleetAuthzRouter) => { .addVersion( { version: API_VERSIONS.public.v1, - validate: {}, + validate: { + request: {}, + response: { + 200: { + body: () => OrphanedPackagePoliciesResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, getOrphanedPackagePolicies ); // Create + // Authz check moved to service here: https://github.com/elastic/kibana/pull/140458 router.versioned .post({ path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN, + description: 'Create package policy', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: CreatePackagePolicyRequestSchema }, + validate: { + request: CreatePackagePolicyRequestSchema, + response: { + 200: { + body: () => CreatePackagePolicyResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + 409: { + body: genericErrorResponse, + }, + }, + }, }, createPackagePolicyHandler ); @@ -130,11 +222,31 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz, getRouteRequiredAuthz('put', PACKAGE_POLICY_API_ROUTES.UPDATE_PATTERN) ).granted, + description: 'Update package policy by ID', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: UpdatePackagePolicyRequestSchema }, + validate: { + request: UpdatePackagePolicyRequestSchema, + response: { + 200: { + body: () => + schema.object({ + item: PackagePolicyResponseSchema, + }), + }, + 400: { + body: genericErrorResponse, + }, + 403: { + body: genericErrorResponse, + }, + }, + }, }, updatePackagePolicyHandler @@ -147,11 +259,25 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz: { integrations: { writeIntegrationPolicies: true }, }, + description: 'Bulk delete package policies', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: DeletePackagePoliciesRequestSchema }, + validate: { + request: DeletePackagePoliciesRequestSchema, + response: { + 200: { + body: () => DeletePackagePoliciesResponseBodySchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, deletePackagePolicyHandler ); @@ -162,11 +288,25 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz: { integrations: { writeIntegrationPolicies: true }, }, + description: 'Delete package policy by ID', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: DeleteOnePackagePolicyRequestSchema }, + validate: { + request: DeleteOnePackagePolicyRequestSchema, + response: { + 200: { + body: () => DeleteOnePackagePolicyResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, deleteOnePackagePolicyHandler ); @@ -178,11 +318,25 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz: { integrations: { writeIntegrationPolicies: true }, }, + description: 'Upgrade package policy to a newer package version', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: UpgradePackagePoliciesRequestSchema }, + validate: { + request: UpgradePackagePoliciesRequestSchema, + response: { + 200: { + body: () => UpgradePackagePoliciesResponseBodySchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, upgradePackagePolicyHandler ); @@ -194,11 +348,25 @@ export const registerRoutes = (router: FleetAuthzRouter) => { fleetAuthz: { integrations: { readIntegrationPolicies: true }, }, + description: 'Dry run package policy upgrade', + options: { + tags: ['oas-tag:Fleet package policies'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: DryRunPackagePoliciesRequestSchema }, + validate: { + request: DryRunPackagePoliciesRequestSchema, + response: { + 200: { + body: () => DryRunPackagePoliciesResponseBodySchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, dryRunUpgradePackagePolicyHandler ); diff --git a/x-pack/plugins/fleet/server/routes/schema/errors.ts b/x-pack/plugins/fleet/server/routes/schema/errors.ts new file mode 100644 index 0000000000000..1d8f0f5d5b92d --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/schema/errors.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +export const genericErrorResponse = () => + schema.object( + { + statusCode: schema.number(), + error: schema.string(), + message: schema.string(), + }, + { + meta: { description: 'Generic Error' }, + } + ); + +export const notFoundResponse = () => + schema.object({ + message: schema.string(), + }); + +export const internalErrorResponse = () => + schema.object( + { + message: schema.string(), + }, + { meta: { description: 'Internal Server Error' } } + ); diff --git a/x-pack/plugins/fleet/server/routes/schema/utils.ts b/x-pack/plugins/fleet/server/routes/schema/utils.ts new file mode 100644 index 0000000000000..2634b58cdcb34 --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/schema/utils.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Type } from '@kbn/config-schema'; +import { schema } from '@kbn/config-schema'; +export const ListResponseSchema = (itemSchema: Type) => + schema.object({ + items: schema.arrayOf(itemSchema), + total: schema.number(), + page: schema.number(), + perPage: schema.number(), + }); diff --git a/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts b/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts index 0a39101db8481..029efc146e09b 100644 --- a/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts +++ b/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts @@ -4,12 +4,20 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { httpServerMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; import { agentPolicyService } from '../../services'; import { getFleetServerPolicies } from '../../services/fleet_server'; -import { getFleetServerOrAgentPolicies, getDownloadSource } from './enrollment_settings_handler'; +import type { FleetRequestHandlerContext } from '../../types'; +import { GetEnrollmentSettingsResponseSchema } from '../../types'; +import { xpackMocks } from '../../mocks'; + +import { + getFleetServerOrAgentPolicies, + getDownloadSource, + getEnrollmentSettingsHandler, +} from './enrollment_settings_handler'; jest.mock('../../services', () => ({ agentPolicyService: { @@ -42,6 +50,27 @@ jest.mock('../../services', () => ({ jest.mock('../../services/fleet_server', () => ({ getFleetServerPolicies: jest.fn(), + hasFleetServersForPolicies: jest.fn().mockResolvedValue(true), +})); + +jest.mock('../../services/fleet_server_host', () => ({ + getFleetServerHostsForAgentPolicy: jest.fn().mockResolvedValue({ + id: 'host-1', + is_default: true, + is_preconfigured: true, + name: 'Host 1', + host_urls: ['http://localhost:8220'], + proxy_id: 'proxy-1', + }), +})); + +jest.mock('../../services/fleet_proxies', () => ({ + getFleetProxy: jest.fn().mockResolvedValue({ + id: 'proxy-1', + name: 'Proxy 1', + url: 'https://proxy-1/', + is_preconfigured: true, + }), })); describe('EnrollmentSettingsHandler utils', () => { @@ -206,5 +235,74 @@ describe('EnrollmentSettingsHandler utils', () => { proxy_id: 'proxy-1', }); }); + + describe('schema validation', () => { + let context: FleetRequestHandlerContext; + let response: ReturnType; + + beforeEach(() => { + context = xpackMocks.createRequestHandlerContext() as unknown as FleetRequestHandlerContext; + response = httpServerMock.createResponseFactory(); + }); + + it('should return valid enrollment settings', async () => { + const fleetServerPolicies = [ + { + id: 'fs-policy-1', + name: 'FS Policy 1', + is_managed: true, + is_default_fleet_server: true, + has_fleet_server: true, + download_source_id: 'source-2', + fleet_server_host_id: undefined, + }, + ]; + (getFleetServerPolicies as jest.Mock).mockResolvedValueOnce(fleetServerPolicies); + const expectedResponse = { + fleet_server: { + has_active: true, + host_proxy: { + id: 'proxy-1', + name: 'Proxy 1', + is_preconfigured: true, + url: 'https://proxy-1/', + }, + + host: { + host_urls: ['http://localhost:8220'], + id: 'host-1', + is_default: true, + is_preconfigured: true, + name: 'Host 1', + proxy_id: 'proxy-1', + }, + policies: [ + { + download_source_id: 'source-2', + fleet_server_host_id: undefined, + has_fleet_server: true, + id: 'fs-policy-1', + is_default_fleet_server: true, + is_managed: true, + name: 'FS Policy 1', + space_ids: undefined, + }, + ], + }, + download_source: { + host: 'https://source-1/', + id: 'source-1', + is_default: true, + name: 'Source 1', + }, + }; + await getEnrollmentSettingsHandler(context, {} as any, response); + expect(response.ok).toHaveBeenCalledWith({ + body: expectedResponse, + }); + const validationResp = GetEnrollmentSettingsResponseSchema.validate(expectedResponse); + expect(validationResp).toEqual(expectedResponse); + }); + }); }); }); diff --git a/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.ts b/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.ts index 17cb27296d9e4..73a7d03a14592 100644 --- a/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.ts +++ b/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.ts @@ -47,7 +47,6 @@ export const getEnrollmentSettingsHandler: FleetRequestHandler< fleet_server_host_id: undefined, download_source_id: undefined, }; - // Check if there is any active fleet server enrolled into the fleet server policies policies if (fleetServerPolicies) { settingsResponse.fleet_server.policies = fleetServerPolicies; diff --git a/x-pack/plugins/fleet/server/routes/settings/index.ts b/x-pack/plugins/fleet/server/routes/settings/index.ts index b9f672627daa7..b101937e45c27 100644 --- a/x-pack/plugins/fleet/server/routes/settings/index.ts +++ b/x-pack/plugins/fleet/server/routes/settings/index.ts @@ -15,9 +15,14 @@ import { GetEnrollmentSettingsRequestSchema, GetSpaceSettingsRequestSchema, PutSpaceSettingsRequestSchema, + SpaceSettingsResponseSchema, + SettingsResponseSchema, + GetEnrollmentSettingsResponseSchema, } from '../../types'; import type { FleetConfigType } from '../../config'; +import { genericErrorResponse, notFoundResponse } from '../schema/errors'; + import { getEnrollmentSettingsHandler } from './enrollment_settings_handler'; import { @@ -45,7 +50,14 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetSpaceSettingsRequestSchema }, + validate: { + request: GetSpaceSettingsRequestSchema, + response: { + 200: { + body: () => SpaceSettingsResponseSchema, + }, + }, + }, }, getSpaceSettingsHandler ); @@ -61,7 +73,14 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: PutSpaceSettingsRequestSchema }, + validate: { + request: PutSpaceSettingsRequestSchema, + response: { + 200: { + body: () => SpaceSettingsResponseSchema, + }, + }, + }, }, putSpaceSettingsHandler ); @@ -74,11 +93,27 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType fleet: { readSettings: true }, }, description: `Get settings`, + options: { + tags: ['oas-tag:Fleet internals'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetSettingsRequestSchema }, + validate: { + request: GetSettingsRequestSchema, + response: { + 200: { + body: () => SettingsResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + 404: { + body: notFoundResponse, + }, + }, + }, }, getSettingsHandler ); @@ -89,11 +124,27 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType fleet: { allSettings: true }, }, description: `Update settings`, + options: { + tags: ['oas-tag:Fleet internals'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: PutSettingsRequestSchema }, + validate: { + request: PutSettingsRequestSchema, + response: { + 200: { + body: () => SettingsResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + 404: { + body: notFoundResponse, + }, + }, + }, }, putSettingsHandler ); @@ -104,11 +155,24 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType return authz.fleet.addAgents || authz.fleet.addFleetServers; }, description: `Get enrollment settings`, + options: { + tags: ['oas-tag:Fleet internals'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetEnrollmentSettingsRequestSchema }, + validate: { + request: GetEnrollmentSettingsRequestSchema, + response: { + 200: { + body: () => GetEnrollmentSettingsResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, getEnrollmentSettingsHandler ); diff --git a/x-pack/plugins/fleet/server/routes/settings/settings_handler.test.ts b/x-pack/plugins/fleet/server/routes/settings/settings_handler.test.ts new file mode 100644 index 0000000000000..0b52c44cde269 --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/settings/settings_handler.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServerMock } from '@kbn/core-http-server-mocks'; + +import { xpackMocks } from '../../mocks'; +import type { FleetRequestHandlerContext } from '../..'; +import { SettingsResponseSchema, SpaceSettingsResponseSchema } from '../../types'; + +import { getSettingsHandler, getSpaceSettingsHandler } from './settings_handler'; + +jest.mock('../../services/spaces/space_settings', () => ({ + getSpaceSettings: jest + .fn() + .mockResolvedValue({ allowed_namespace_prefixes: [], managed_by: 'kibana' }), + saveSpaceSettings: jest.fn(), +})); + +jest.mock('../../services', () => ({ + settingsService: { + getSettings: jest.fn().mockResolvedValue({ + id: '1', + version: '1', + preconfigured_fields: ['fleet_server_hosts'], + secret_storage_requirements_met: true, + output_secret_storage_requirements_met: true, + has_seen_add_data_notice: true, + fleet_server_hosts: ['http://localhost:8220'], + prerelease_integrations_enabled: true, + }), + }, + appContextService: { + getLogger: jest.fn().mockReturnValue({ error: jest.fn() }), + getInternalUserSOClientWithoutSpaceExtension: jest.fn(), + }, + agentPolicyService: { + get: jest.fn(), + getByIDs: jest.fn(), + }, +})); + +describe('SettingsHandler', () => { + let context: FleetRequestHandlerContext; + let response: ReturnType; + + beforeEach(() => { + context = xpackMocks.createRequestHandlerContext() as unknown as FleetRequestHandlerContext; + response = httpServerMock.createResponseFactory(); + }); + + it('should return valid space settings', async () => { + await getSpaceSettingsHandler(context, {} as any, response); + const expectedResponse = { item: { allowed_namespace_prefixes: [], managed_by: 'kibana' } }; + expect(response.ok).toHaveBeenCalledWith({ + body: expectedResponse, + }); + const validationResp = SpaceSettingsResponseSchema.validate(expectedResponse); + expect(validationResp).toEqual(expectedResponse); + }); + + it('should return valid settings', async () => { + await getSettingsHandler(context, {} as any, response); + const expectedResponse = { + item: { + id: '1', + version: '1', + preconfigured_fields: ['fleet_server_hosts'], + secret_storage_requirements_met: true, + output_secret_storage_requirements_met: true, + has_seen_add_data_notice: true, + fleet_server_hosts: ['http://localhost:8220'], + prerelease_integrations_enabled: true, + }, + }; + expect(response.ok).toHaveBeenCalledWith({ + body: expectedResponse, + }); + const validationResp = SettingsResponseSchema.validate(expectedResponse); + expect(validationResp).toEqual(expectedResponse); + }); +}); diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts index 6923d00c18222..79544e7a4e932 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts @@ -25,6 +25,7 @@ import { hasFleetServers } from '../../services/fleet_server'; import { createFleetAuthzMock } from '../../../common/mocks'; import { fleetSetupHandler, getFleetStatusHandler } from './handlers'; +import { FleetSetupResponseSchema, GetAgentsSetupResponseSchema } from '.'; jest.mock('../../services/setup', () => { return { @@ -94,6 +95,8 @@ describe('FleetSetupHandler', () => { }; expect(response.customError).toHaveBeenCalledTimes(0); expect(response.ok).toHaveBeenCalledWith({ body: expectedBody }); + const validationResp = FleetSetupResponseSchema.validate(expectedBody); + expect(validationResp).toEqual(expectedBody); }); it('POST /setup fails w/500 on custom error', async () => { @@ -209,6 +212,8 @@ describe('FleetStatusHandler', () => { }; expect(response.customError).toHaveBeenCalledTimes(0); expect(response.ok).toHaveBeenCalledWith({ body: expectedBody }); + const validationResp = GetAgentsSetupResponseSchema.validate(expectedBody); + expect(validationResp).toEqual(expectedBody); }); it('POST /status w/200 with fleet server standalone', async () => { diff --git a/x-pack/plugins/fleet/server/routes/setup/index.ts b/x-pack/plugins/fleet/server/routes/setup/index.ts index 7052aacfc329d..4b6fd2316832d 100644 --- a/x-pack/plugins/fleet/server/routes/setup/index.ts +++ b/x-pack/plugins/fleet/server/routes/setup/index.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { schema } from '@kbn/config-schema'; import type { FleetAuthzRouter } from '../../services/security'; @@ -12,8 +13,28 @@ import { API_VERSIONS } from '../../../common/constants'; import type { FleetConfigType } from '../../../common/types'; +import { genericErrorResponse, internalErrorResponse } from '../schema/errors'; + import { getFleetStatusHandler, fleetSetupHandler } from './handlers'; +export const FleetSetupResponseSchema = schema.object( + { + isInitialized: schema.boolean(), + nonFatalErrors: schema.arrayOf( + schema.object({ + name: schema.string(), + message: schema.string(), + }) + ), + }, + { + meta: { + description: + "A summary of the result of Fleet's `setup` lifecycle. If `isInitialized` is true, Fleet is ready to accept agent enrollment. `nonFatalErrors` may include useful insight into non-blocking issues with Fleet setup.", + }, + } +); + export const registerFleetSetupRoute = (router: FleetAuthzRouter) => { router.versioned .post({ @@ -22,16 +43,59 @@ export const registerFleetSetupRoute = (router: FleetAuthzRouter) => { fleet: { setup: true }, }, description: `Initiate Fleet setup`, + options: { + tags: ['oas-tag:Fleet internals'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: false, + validate: { + request: {}, + response: { + 200: { + body: () => FleetSetupResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + 500: { + body: internalErrorResponse, + }, + }, + }, }, fleetSetupHandler ); }; +export const GetAgentsSetupResponseSchema = schema.object( + { + isReady: schema.boolean(), + missing_requirements: schema.arrayOf( + schema.oneOf([ + schema.literal('security_required'), + schema.literal('tls_required'), + schema.literal('api_keys'), + schema.literal('fleet_admin_user'), + schema.literal('fleet_server'), + ]) + ), + missing_optional_features: schema.arrayOf( + schema.oneOf([schema.literal('encrypted_saved_object_encryption_key_required')]) + ), + package_verification_key_id: schema.maybe(schema.string()), + is_space_awareness_enabled: schema.maybe(schema.boolean()), + is_secrets_storage_enabled: schema.maybe(schema.boolean()), + }, + { + meta: { + description: + 'A summary of the agent setup status. `isReady` indicates whether the setup is ready. If the setup is not ready, `missing_requirements` lists which requirements are missing.', + }, + } +); + // That route is used by agent to setup Fleet export const registerCreateFleetSetupRoute = (router: FleetAuthzRouter) => { router.versioned @@ -40,11 +104,25 @@ export const registerCreateFleetSetupRoute = (router: FleetAuthzRouter) => { fleetAuthz: { fleet: { setup: true }, }, + description: `Initiate agent setup`, + options: { + tags: ['oas-tag:Elastic Agents'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: false, + validate: { + request: {}, + response: { + 200: { + body: () => FleetSetupResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, fleetSetupHandler ); @@ -57,11 +135,25 @@ export const registerGetFleetStatusRoute = (router: FleetAuthzRouter) => { fleetAuthz: { fleet: { setup: true }, }, + description: `Get agent setup info`, + options: { + tags: ['oas-tag:Elastic Agents'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: false, + validate: { + request: {}, + response: { + 200: { + body: () => GetAgentsSetupResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, getFleetStatusHandler ); diff --git a/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/handler.ts b/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/handler.ts index 99c349899aaa6..60e0c40c03f40 100644 --- a/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/handler.ts +++ b/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/handler.ts @@ -9,18 +9,38 @@ import type { TypeOf } from '@kbn/config-schema'; import { createStandaloneAgentApiKey } from '../../services/api_keys'; import type { FleetRequestHandler, PostStandaloneAgentAPIKeyRequestSchema } from '../../types'; +import { + INDEX_PRIVILEGES, + canCreateStandaloneAgentApiKey, +} from '../../services/api_keys/create_standalone_agent_api_key'; +import { FleetUnauthorizedError, defaultFleetErrorHandler } from '../../errors'; export const createStandaloneAgentApiKeyHandler: FleetRequestHandler< undefined, undefined, TypeOf > = async (context, request, response) => { - const coreContext = await context.core; - const esClient = coreContext.elasticsearch.client.asCurrentUser; - const key = await createStandaloneAgentApiKey(esClient, request.body.name); - return response.ok({ - body: { - item: key, - }, - }); + try { + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const canCreate = await canCreateStandaloneAgentApiKey(esClient); + + if (!canCreate) { + throw new FleetUnauthorizedError( + `Missing permissions to create standalone API key, You need ${INDEX_PRIVILEGES.privileges.join( + ', ' + )} for indices ${INDEX_PRIVILEGES.names.join(', ')}` + ); + } + + const key = await createStandaloneAgentApiKey(esClient, request.body.name); + + return response.ok({ + body: { + item: key, + }, + }); + } catch (error) { + return defaultFleetErrorHandler({ error, response }); + } }; diff --git a/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/index.ts b/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/index.ts index 9255f058aee46..f0103c23e65dd 100644 --- a/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/index.ts +++ b/x-pack/plugins/fleet/server/routes/standalone_agent_api_key/index.ts @@ -21,13 +21,15 @@ export const registerRoutes = (router: FleetAuthzRouter) => { path: CREATE_STANDALONE_AGENT_API_KEY_ROUTE, access: 'internal', fleetAuthz: { - fleet: { all: true }, + fleet: { addAgents: true }, }, }) .addVersion( { version: API_VERSIONS.internal.v1, - validate: { request: PostStandaloneAgentAPIKeyRequestSchema }, + validate: { + request: PostStandaloneAgentAPIKeyRequestSchema, + }, }, createStandaloneAgentApiKeyHandler ); diff --git a/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts b/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts index b51f19291c58a..d7432fa7f2f51 100644 --- a/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts @@ -28,9 +28,11 @@ import type { FleetRequestHandlerContext } from '../..'; import type { MockedFleetAppContext } from '../../mocks'; import { createAppContextStartContractMock, xpackMocks } from '../../mocks'; import { agentPolicyService, appContextService } from '../../services'; -import type { - GetUninstallTokenRequestSchema, - GetUninstallTokensMetadataRequestSchema, +import { + GetUninstallTokensMetadataResponseSchema, + type GetUninstallTokenRequestSchema, + type GetUninstallTokensMetadataRequestSchema, + GetUninstallTokenResponseSchema, } from '../../types/rest_spec/uninstall_token'; import { createAgentPolicyMock } from '../../../common/mocks'; @@ -116,6 +118,10 @@ describe('uninstall token handlers', () => { expect(response.ok).toHaveBeenCalledWith({ body: uninstallTokensResponseFixture, }); + const validateResp = GetUninstallTokensMetadataResponseSchema.validate( + uninstallTokensResponseFixture + ); + expect(validateResp).toEqual(uninstallTokensResponseFixture); }); it('should return internal error when uninstallTokenService throws error', async () => { @@ -131,18 +137,19 @@ describe('uninstall token handlers', () => { }); describe('getUninstallTokenHandler', () => { - const uninstallTokenFixture: UninstallToken = { - id: 'id-1', - policy_id: 'policy-id-1', - policy_name: null, - created_at: '2023-06-15T16:46:48.274Z', - token: '123456789', - }; + let uninstallTokenFixture: UninstallToken; let getTokenMock: jest.Mock; let request: KibanaRequest>; beforeEach(async () => { + uninstallTokenFixture = { + id: 'id-1', + policy_id: 'policy-id-1', + policy_name: null, + created_at: '2023-06-15T16:46:48.274Z', + token: '123456789', + }; const uninstallTokenService = (await context.fleet).uninstallTokenService.asCurrentUser; getTokenMock = uninstallTokenService.getToken as jest.Mock; @@ -165,6 +172,10 @@ describe('uninstall token handlers', () => { item: uninstallTokenFixture, }, }); + const validateResp = GetUninstallTokenResponseSchema.validate({ + item: uninstallTokenFixture, + }); + expect(validateResp).toEqual({ item: uninstallTokenFixture }); }); it('should return internal error when uninstallTokenService throws error', async () => { diff --git a/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.ts b/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.ts index 24d85b8d14250..2eb9a83456845 100644 --- a/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.ts @@ -84,7 +84,6 @@ export const getUninstallTokenHandler: FleetRequestHandler< body: { message: `Uninstall Token not found with id ${uninstallTokenId}` }, }); } - const body: GetUninstallTokenResponse = { item: token, }; diff --git a/x-pack/plugins/fleet/server/routes/uninstall_token/index.ts b/x-pack/plugins/fleet/server/routes/uninstall_token/index.ts index 9fb91b45fa373..a90dd678e99dd 100644 --- a/x-pack/plugins/fleet/server/routes/uninstall_token/index.ts +++ b/x-pack/plugins/fleet/server/routes/uninstall_token/index.ts @@ -4,16 +4,21 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import { UNINSTALL_TOKEN_ROUTES, API_VERSIONS } from '../../../common/constants'; import type { FleetConfigType } from '../../config'; import type { FleetAuthzRouter } from '../../services/security'; import { GetUninstallTokenRequestSchema, + GetUninstallTokenResponseSchema, GetUninstallTokensMetadataRequestSchema, + GetUninstallTokensMetadataResponseSchema, } from '../../types/rest_spec/uninstall_token'; import { parseExperimentalConfigValue } from '../../../common/experimental_features'; +import { genericErrorResponse } from '../schema/errors'; + import { getUninstallTokenHandler, getUninstallTokensMetadataHandler } from './handlers'; export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType) => { @@ -26,11 +31,25 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType fleetAuthz: { fleet: { allAgents: true }, }, + description: 'List metadata for latest uninstall tokens per agent policy', + options: { + tags: ['oas-tag:Fleet uninstall tokens'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetUninstallTokensMetadataRequestSchema }, + validate: { + request: GetUninstallTokensMetadataRequestSchema, + response: { + 200: { + body: () => GetUninstallTokensMetadataResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, getUninstallTokensMetadataHandler ); @@ -41,11 +60,25 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType fleetAuthz: { fleet: { allAgents: true }, }, + description: 'Get one decrypted uninstall token by its ID', + options: { + tags: ['oas-tag:Fleet uninstall tokens'], + }, }) .addVersion( { version: API_VERSIONS.public.v1, - validate: { request: GetUninstallTokenRequestSchema }, + validate: { + request: GetUninstallTokenRequestSchema, + response: { + 200: { + body: () => GetUninstallTokenResponseSchema, + }, + 400: { + body: genericErrorResponse, + }, + }, + }, }, getUninstallTokenHandler ); diff --git a/x-pack/plugins/fleet/server/services/api_keys/create_standalone_agent_api_key.ts b/x-pack/plugins/fleet/server/services/api_keys/create_standalone_agent_api_key.ts index 011d8dfe8ec82..51cf0b7849a03 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/create_standalone_agent_api_key.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/create_standalone_agent_api_key.ts @@ -7,6 +7,21 @@ import type { ElasticsearchClient } from '@kbn/core/server'; +export const CLUSTER_PRIVILEGES = ['monitor']; + +export const INDEX_PRIVILEGES = { + names: ['logs-*-*', 'metrics-*-*', 'traces-*-*', 'synthetics-*-*'], + privileges: ['auto_configure', 'create_doc'], +}; + +export async function canCreateStandaloneAgentApiKey(esClient: ElasticsearchClient) { + const res = await esClient.security.hasPrivileges({ + cluster: CLUSTER_PRIVILEGES, + index: [INDEX_PRIVILEGES], + }); + + return res.has_all_requested; +} export function createStandaloneAgentApiKey(esClient: ElasticsearchClient, name: string) { // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent return esClient.security.createApiKey({ @@ -17,13 +32,8 @@ export function createStandaloneAgentApiKey(esClient: ElasticsearchClient, name: }, role_descriptors: { standalone_agent: { - cluster: ['monitor'], - indices: [ - { - names: ['logs-*-*', 'metrics-*-*', 'traces-*-*', 'synthetics-*-*'], - privileges: ['auto_configure', 'create_doc'], - }, - ], + cluster: CLUSTER_PRIVILEGES, + indices: [INDEX_PRIVILEGES], }, }, }, diff --git a/x-pack/plugins/fleet/server/services/settings.ts b/x-pack/plugins/fleet/server/services/settings.ts index 68829b734eeaf..394a2365f3c03 100644 --- a/x-pack/plugins/fleet/server/services/settings.ts +++ b/x-pack/plugins/fleet/server/services/settings.ts @@ -36,7 +36,15 @@ export async function getSettings(soClient: SavedObjectsClientContract): Promise return { id: settingsSo.id, version: settingsSo.version, - ...settingsSo.attributes, + secret_storage_requirements_met: settingsSo.attributes.secret_storage_requirements_met, + output_secret_storage_requirements_met: + settingsSo.attributes.output_secret_storage_requirements_met, + has_seen_add_data_notice: settingsSo.attributes.has_seen_add_data_notice, + prerelease_integrations_enabled: settingsSo.attributes.prerelease_integrations_enabled, + use_space_awareness_migration_status: + settingsSo.attributes.use_space_awareness_migration_status, + use_space_awareness_migration_started_at: + settingsSo.attributes.use_space_awareness_migration_started_at, fleet_server_hosts: fleetServerHosts.items.flatMap((item) => item.host_urls), preconfigured_fields: getConfigFleetServerHosts() ? ['fleet_server_hosts'] : [], }; diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index e105bc82b27db..68d3a089c2342 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -16,15 +16,24 @@ export const PackagePolicyNamespaceSchema = schema.string({ return namespaceValidation.error; } }, + meta: { + description: + "The package policy namespace. Leave blank to inherit the agent policy's namespace.", + }, }); -const ConfigRecordSchema = schema.recordOf( +export const ConfigRecordSchema = schema.recordOf( schema.string(), schema.object({ type: schema.maybe(schema.string()), value: schema.maybe(schema.any()), frozen: schema.maybe(schema.boolean()), - }) + }), + { + meta: { + description: 'Package variable (see integration documentation for more information)', + }, + } ); const PackagePolicyStreamsSchema = { @@ -50,33 +59,18 @@ const PackagePolicyStreamsSchema = { ), }), vars: schema.maybe(ConfigRecordSchema), - config: schema.maybe( - schema.recordOf( - schema.string(), - schema.object({ - type: schema.maybe(schema.string()), - value: schema.maybe(schema.any()), - }) - ) - ), + config: schema.maybe(ConfigRecordSchema), compiled_stream: schema.maybe(schema.any()), }; -const PackagePolicyInputsSchema = { +export const PackagePolicyInputsSchema = { + id: schema.maybe(schema.string()), type: schema.string(), policy_template: schema.maybe(schema.string()), enabled: schema.boolean(), keep_enabled: schema.maybe(schema.boolean()), vars: schema.maybe(ConfigRecordSchema), - config: schema.maybe( - schema.recordOf( - schema.string(), - schema.object({ - type: schema.maybe(schema.string()), - value: schema.maybe(schema.any()), - }) - ) - ), + config: schema.maybe(ConfigRecordSchema), streams: schema.arrayOf(schema.object(PackagePolicyStreamsSchema)), }; @@ -92,44 +86,91 @@ const ExperimentalDataStreamFeatures = schema.arrayOf( }) ); -const PackagePolicyBaseSchema = { - name: schema.string(), - description: schema.maybe(schema.string()), +export const PackagePolicyPackageSchema = schema.object({ + name: schema.string({ + meta: { + description: 'Package name', + }, + }), + title: schema.maybe(schema.string()), + version: schema.string({ + meta: { + description: 'Package version', + }, + }), + experimental_data_stream_features: schema.maybe(ExperimentalDataStreamFeatures), + requires_root: schema.maybe(schema.boolean()), +}); + +export const PackagePolicyBaseSchema = { + name: schema.string({ + meta: { + description: 'Package policy name (should be unique)', + }, + }), + description: schema.maybe( + schema.string({ + meta: { + description: 'Package policy description', + }, + }) + ), namespace: schema.maybe(PackagePolicyNamespaceSchema), - policy_id: schema.nullable(schema.maybe(schema.string())), - policy_ids: schema.maybe(schema.arrayOf(schema.string())), - output_id: schema.nullable(schema.maybe(schema.string())), + policy_id: schema.maybe( + schema.oneOf([ + schema.literal(null), + schema.string({ + meta: { + description: 'Agent policy ID where that package policy will be added', + deprecated: true, + }, + }), + ]) + ), + policy_ids: schema.maybe( + schema.arrayOf( + schema.string({ + meta: { + description: 'Agent policy IDs where that package policy will be added', + }, + }) + ) + ), + output_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), enabled: schema.boolean(), is_managed: schema.maybe(schema.boolean()), - package: schema.maybe( - schema.object({ - name: schema.string(), - title: schema.string(), - version: schema.string(), - experimental_data_stream_features: schema.maybe(ExperimentalDataStreamFeatures), - requires_root: schema.maybe(schema.boolean()), - }) - ), + package: schema.maybe(PackagePolicyPackageSchema), + inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)), vars: schema.maybe(ConfigRecordSchema), overrides: schema.maybe( - schema.nullable( - schema.object({ - inputs: schema.maybe( - schema.recordOf(schema.string(), schema.any(), { - validate: (val) => { - if ( - Object.keys(val).some( - (key) => key.match(/^compiled_inputs(\.)?/) || key.match(/^compiled_stream(\.)?/) - ) - ) { - return 'Overrides of compiled_inputs and compiled_stream are not allowed'; - } - }, - }) - ), - }) - ) + schema.oneOf([ + schema.literal(null), + schema.object( + { + inputs: schema.maybe( + schema.recordOf(schema.string(), schema.any(), { + validate: (val) => { + if ( + Object.keys(val).some( + (key) => + key.match(/^compiled_inputs(\.)?/) || key.match(/^compiled_stream(\.)?/) + ) + ) { + return 'Overrides of compiled_inputs and compiled_stream are not allowed'; + } + }, + }) + ), + }, + { + meta: { + description: + 'Override settings that are defined in the package policy. The override option should be used only in unusual circumstances and not as a routine procedure.', + }, + } + ), + ]) ), }; @@ -142,15 +183,7 @@ export const NewPackagePolicySchema = schema.object({ const CreatePackagePolicyProps = { ...PackagePolicyBaseSchema, enabled: schema.maybe(schema.boolean()), - package: schema.maybe( - schema.object({ - name: schema.string(), - title: schema.maybe(schema.string()), - version: schema.string(), - experimental_data_stream_features: schema.maybe(ExperimentalDataStreamFeatures), - requires_root: schema.maybe(schema.boolean()), - }) - ), + package: schema.maybe(PackagePolicyPackageSchema), inputs: schema.arrayOf( schema.object({ ...PackagePolicyInputsSchema, @@ -161,11 +194,24 @@ const CreatePackagePolicyProps = { export const CreatePackagePolicyRequestBodySchema = schema.object({ ...CreatePackagePolicyProps, - id: schema.maybe(schema.string()), - force: schema.maybe(schema.boolean()), + id: schema.maybe( + schema.string({ + meta: { + description: 'Package policy unique identifier', + }, + }) + ), + force: schema.maybe( + schema.boolean({ + meta: { + description: + 'Force package policy creation even if package is not verified, or if the agent policy is managed.', + }, + }) + ), }); -const SimplifiedVarsSchema = schema.recordOf( +export const SimplifiedVarsSchema = schema.recordOf( schema.string(), schema.nullable( schema.oneOf([ @@ -180,6 +226,55 @@ const SimplifiedVarsSchema = schema.recordOf( isSecretRef: schema.boolean(), }), ]) + ), + { + meta: { + description: + 'Input/stream level variable (see integration documentation for more information)', + }, + } +); + +export const SimplifiedPackagePolicyInputsSchema = schema.maybe( + schema.recordOf( + schema.string(), + schema.object({ + enabled: schema.maybe( + schema.boolean({ + meta: { + description: 'enable or disable that input, (default to true)', + }, + }) + ), + vars: schema.maybe(SimplifiedVarsSchema), + streams: schema.maybe( + schema.recordOf( + schema.string(), + schema.object({ + enabled: schema.maybe( + schema.boolean({ + meta: { + description: 'enable or disable that stream, (default to true)', + }, + }) + ), + vars: schema.maybe(SimplifiedVarsSchema), + }), + { + meta: { + description: + 'Input streams (see integration documentation to know what streams are available)', + }, + } + ) + ), + }), + { + meta: { + description: + 'Package policy inputs (see integration documentation to know what inputs are available)', + }, + } ) ); @@ -188,26 +283,9 @@ export const SimplifiedPackagePolicyBaseSchema = schema.object({ name: schema.string(), description: schema.maybe(schema.string()), namespace: schema.maybe(schema.string()), - output_id: schema.nullable(schema.maybe(schema.string())), + output_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), vars: schema.maybe(SimplifiedVarsSchema), - inputs: schema.maybe( - schema.recordOf( - schema.string(), - schema.object({ - enabled: schema.maybe(schema.boolean()), - vars: schema.maybe(SimplifiedVarsSchema), - streams: schema.maybe( - schema.recordOf( - schema.string(), - schema.object({ - enabled: schema.maybe(schema.boolean()), - vars: schema.maybe(SimplifiedVarsSchema), - }) - ) - ), - }) - ) - ), + inputs: SimplifiedPackagePolicyInputsSchema, }); export const SimplifiedPackagePolicyPreconfiguredSchema = SimplifiedPackagePolicyBaseSchema.extends( @@ -221,15 +299,10 @@ export const SimplifiedPackagePolicyPreconfiguredSchema = SimplifiedPackagePolic export const SimplifiedCreatePackagePolicyRequestBodySchema = SimplifiedPackagePolicyBaseSchema.extends({ - policy_id: schema.nullable(schema.maybe(schema.string())), + policy_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), policy_ids: schema.maybe(schema.arrayOf(schema.string())), force: schema.maybe(schema.boolean()), - package: schema.object({ - name: schema.string(), - version: schema.string(), - experimental_data_stream_features: schema.maybe(ExperimentalDataStreamFeatures), - requires_root: schema.maybe(schema.boolean()), - }), + package: PackagePolicyPackageSchema, }); export const UpdatePackagePolicyRequestBodySchema = schema.object({ @@ -261,15 +334,19 @@ export const PackagePolicySchema = schema.object({ updated_by: schema.string(), created_at: schema.string(), created_by: schema.string(), - elasticsearch: schema.maybe( - schema.object({ - privileges: schema.maybe( - schema.object({ - cluster: schema.maybe(schema.arrayOf(schema.string())), - }) - ), - }) - ), + elasticsearch: schema + .maybe( + schema.object({ + privileges: schema.maybe( + schema.object({ + cluster: schema.maybe(schema.arrayOf(schema.string())), + }) + ), + }) + ) + .extendsDeep({ + unknowns: 'allow', + }), inputs: schema.arrayOf( schema.object({ ...PackagePolicyInputsSchema, @@ -284,3 +361,46 @@ export const PackagePolicySchema = schema.object({ ) ), }); + +export const PackagePolicyResponseSchema = PackagePolicySchema.extends({ + vars: schema.maybe(schema.oneOf([ConfigRecordSchema, schema.maybe(SimplifiedVarsSchema)])), + inputs: schema.oneOf([ + schema.arrayOf( + schema.object({ + ...PackagePolicyInputsSchema, + compiled_input: schema.maybe(schema.any()), + }) + ), + SimplifiedPackagePolicyInputsSchema, + ]), + spaceIds: schema.maybe(schema.arrayOf(schema.string())), + agents: schema.maybe(schema.number()), +}); + +export const OrphanedPackagePoliciesResponseSchema = schema.object({ + items: schema.arrayOf(PackagePolicyResponseSchema), + total: schema.number(), +}); + +export const DryRunPackagePolicySchema = schema.object({ + ...PackagePolicyBaseSchema, + id: schema.maybe(schema.string()), + force: schema.maybe(schema.boolean()), + errors: schema.maybe( + schema.arrayOf( + schema.object({ + message: schema.string(), + key: schema.maybe(schema.string()), + }) + ) + ), + missingVars: schema.maybe(schema.arrayOf(schema.string())), +}); + +export const PackagePolicyStatusResponseSchema = schema.object({ + id: schema.string(), + success: schema.boolean(), + name: schema.maybe(schema.string()), + statusCode: schema.maybe(schema.number()), + body: schema.maybe(schema.object({ message: schema.string() })), +}); diff --git a/x-pack/plugins/fleet/server/types/rest_spec/common.ts b/x-pack/plugins/fleet/server/types/rest_spec/common.ts index 0c5f16ff87f90..2be083d677dd3 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/common.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/common.ts @@ -23,7 +23,10 @@ export const ListWithKuerySchema = schema.object({ }); export const BulkRequestBodySchema = schema.object({ - ids: schema.arrayOf(schema.string(), { minSize: 1 }), + ids: schema.arrayOf(schema.string(), { + minSize: 1, + meta: { description: 'list of package policy ids' }, + }), ignoreMissing: schema.maybe(schema.boolean()), }); diff --git a/x-pack/plugins/fleet/server/types/rest_spec/package_policy.ts b/x-pack/plugins/fleet/server/types/rest_spec/package_policy.ts index 88b4452a5fe7a..1cf332efba843 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/package_policy.ts @@ -9,6 +9,10 @@ import { schema } from '@kbn/config-schema'; import { CreatePackagePolicyRequestBodySchema, + DryRunPackagePolicySchema, + PackagePolicyPackageSchema, + PackagePolicyResponseSchema, + PackagePolicyStatusResponseSchema, SimplifiedCreatePackagePolicyRequestBodySchema, UpdatePackagePolicyRequestBodySchema, } from '../models'; @@ -59,6 +63,10 @@ export const BulkGetPackagePoliciesRequestSchema = { }), }; +export const BulkGetPackagePoliciesResponseBodySchema = schema.object({ + items: schema.arrayOf(PackagePolicyResponseSchema), +}); + export const GetOnePackagePolicyRequestSchema = { params: schema.object({ packagePolicyId: schema.string(), @@ -71,10 +79,14 @@ export const GetOnePackagePolicyRequestSchema = { }; export const CreatePackagePolicyRequestSchema = { - body: schema.oneOf([ - CreatePackagePolicyRequestBodySchema, - SimplifiedCreatePackagePolicyRequestBodySchema, - ]), + body: schema.oneOf( + [CreatePackagePolicyRequestBodySchema, SimplifiedCreatePackagePolicyRequestBodySchema], + { + meta: { + description: 'You should use inputs as an object and not use the deprecated inputs array.', + }, + } + ), query: schema.object({ format: schema.maybe( schema.oneOf([schema.literal(inputsFormat.Simplified), schema.literal(inputsFormat.Legacy)]) @@ -82,6 +94,10 @@ export const CreatePackagePolicyRequestSchema = { }), }; +export const CreatePackagePolicyResponseSchema = schema.object({ + item: PackagePolicyResponseSchema, +}); + export const UpdatePackagePolicyRequestSchema = { ...GetOnePackagePolicyRequestSchema, body: schema.oneOf([ @@ -102,6 +118,25 @@ export const DeletePackagePoliciesRequestSchema = { }), }; +export const DeletePackagePoliciesResponseBodySchema = schema.arrayOf( + PackagePolicyStatusResponseSchema.extends({ + policy_id: schema.maybe( + schema.oneOf([ + schema.literal(null), + schema.string({ + meta: { + description: 'Use `policy_ids` instead', + deprecated: true, + }, + }), + ]) + ), + policy_ids: schema.arrayOf(schema.string()), + output_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + package: PackagePolicyPackageSchema, + }) +); + export const DeleteOnePackagePolicyRequestSchema = { params: schema.object({ packagePolicyId: schema.string(), @@ -111,15 +146,104 @@ export const DeleteOnePackagePolicyRequestSchema = { }), }; +export const DeleteOnePackagePolicyResponseSchema = schema.object({ + id: schema.string(), +}); + export const UpgradePackagePoliciesRequestSchema = { body: schema.object({ packagePolicyIds: schema.arrayOf(schema.string()), }), }; +export const UpgradePackagePoliciesResponseBodySchema = schema.arrayOf( + PackagePolicyStatusResponseSchema +); + export const DryRunPackagePoliciesRequestSchema = { body: schema.object({ packagePolicyIds: schema.arrayOf(schema.string()), packageVersion: schema.maybe(schema.string()), }), }; + +export const DryRunPackagePoliciesResponseBodySchema = schema.arrayOf( + schema.object({ + name: schema.maybe(schema.string()), + statusCode: schema.maybe(schema.number()), + body: schema.maybe(schema.object({ message: schema.string() })), + hasErrors: schema.boolean(), + diff: schema.maybe( + schema.arrayOf( + schema.oneOf([ + PackagePolicyResponseSchema.extends({ + id: schema.maybe(schema.string()), + }), + DryRunPackagePolicySchema, + ]) + ) + ), + agent_diff: schema.maybe( + schema.arrayOf( + schema.arrayOf( + schema + .object({ + id: schema.string(), + name: schema.string(), + revision: schema.number(), + type: schema.string(), + data_stream: schema.object({ + namespace: schema.string(), + }), + use_output: schema.string(), + package_policy_id: schema.string(), + meta: schema.maybe( + schema.object({ + package: schema + .object({ + name: schema.string(), + version: schema.string(), + }) + .extendsDeep({ + // equivalent of allowing extra keys like `[key: string]: any;` + unknowns: 'allow', + }), + }) + ), + streams: schema.maybe( + schema.arrayOf( + schema + .object({ + id: schema.string(), + data_stream: schema.object({ + dataset: schema.string(), + type: schema.maybe(schema.string()), + }), + }) + .extendsDeep({ + unknowns: 'allow', + }) + ) + ), + processors: schema.maybe( + schema.arrayOf( + schema.object({ + add_fields: schema.object({ + target: schema.string(), + fields: schema.recordOf( + schema.string(), + schema.oneOf([schema.string(), schema.number()]) + ), + }), + }) + ) + ), + }) + .extendsDeep({ + unknowns: 'allow', + }) + ) + ) + ), + }) +); diff --git a/x-pack/plugins/fleet/server/types/rest_spec/settings.ts b/x-pack/plugins/fleet/server/types/rest_spec/settings.ts index 10db7b0f4def7..6553d2e976bed 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/settings.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/settings.ts @@ -41,6 +41,30 @@ export const PutSettingsRequestSchema = { export const GetSpaceSettingsRequestSchema = {}; +export const SpaceSettingsResponseSchema = schema.object({ + item: schema.object({ + managed_by: schema.maybe(schema.string()), + allowed_namespace_prefixes: schema.arrayOf(schema.string()), + }), +}); + +export const SettingsResponseSchema = schema.object({ + item: schema.object({ + has_seen_add_data_notice: schema.maybe(schema.boolean()), + fleet_server_hosts: schema.maybe(schema.arrayOf(schema.string())), + prerelease_integrations_enabled: schema.boolean(), + id: schema.string(), + version: schema.maybe(schema.string()), + preconfigured_fields: schema.maybe(schema.arrayOf(schema.literal('fleet_server_hosts'))), + secret_storage_requirements_met: schema.maybe(schema.boolean()), + output_secret_storage_requirements_met: schema.maybe(schema.boolean()), + use_space_awareness_migration_status: schema.maybe( + schema.oneOf([schema.literal('pending'), schema.literal('success'), schema.literal('error')]) + ), + use_space_awareness_migration_started_at: schema.maybe(schema.string()), + }), +}); + export const PutSpaceSettingsRequestSchema = { body: schema.object({ allowed_namespace_prefixes: schema.maybe( @@ -64,3 +88,70 @@ export const GetEnrollmentSettingsRequestSchema = { }) ), }; + +export const GetEnrollmentSettingsResponseSchema = schema.object({ + fleet_server: schema.object({ + policies: schema.arrayOf( + schema.object({ + id: schema.string(), + name: schema.string(), + is_managed: schema.boolean(), + is_default_fleet_server: schema.maybe(schema.boolean()), + has_fleet_server: schema.maybe(schema.boolean()), + fleet_server_host_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + download_source_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + space_ids: schema.maybe(schema.arrayOf(schema.string())), + }) + ), + has_active: schema.boolean(), + host: schema.maybe( + schema.object({ + id: schema.string(), + name: schema.string(), + host_urls: schema.arrayOf(schema.string()), + is_default: schema.boolean(), + is_preconfigured: schema.boolean(), + is_internal: schema.maybe(schema.boolean()), + proxy_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + }) + ), + host_proxy: schema.maybe( + schema.object({ + id: schema.string(), + proxy_headers: schema.maybe( + schema.recordOf( + schema.string(), + schema.oneOf([schema.string(), schema.number(), schema.boolean()]) + ) + ), + name: schema.string(), + url: schema.string(), + certificate_authorities: schema.maybe( + schema.oneOf([schema.literal(null), schema.string()]) + ), + certificate: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + certificate_key: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + is_preconfigured: schema.boolean(), + }) + ), + }), + download_source: schema.maybe( + schema.object({ + id: schema.string(), + name: schema.string(), + host: schema.string(), + is_default: schema.boolean(), + proxy_id: schema.maybe( + schema.oneOf([ + schema.literal(null), + schema.string({ + meta: { + description: + 'The ID of the proxy to use for this download source. See the proxies API for more information.', + }, + }), + ]) + ), + }) + ), +}); diff --git a/x-pack/plugins/fleet/server/types/rest_spec/uninstall_token.ts b/x-pack/plugins/fleet/server/types/rest_spec/uninstall_token.ts index 924e1da2cb9e8..7915e5bbc8b07 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/uninstall_token.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/uninstall_token.ts @@ -6,17 +6,48 @@ */ import { schema } from '@kbn/config-schema'; +import { ListResponseSchema } from '../../routes/schema/utils'; + export const GetUninstallTokensMetadataRequestSchema = { query: schema.object({ - policyId: schema.maybe(schema.string({ maxLength: 50 })), + policyId: schema.maybe( + schema.string({ + maxLength: 50, + meta: { description: 'Partial match filtering for policy IDs' }, + }) + ), search: schema.maybe(schema.string({ maxLength: 50 })), - perPage: schema.maybe(schema.number({ defaultValue: 20, min: 5 })), + perPage: schema.maybe( + schema.number({ + defaultValue: 20, + min: 5, + meta: { description: 'The number of items to return' }, + }) + ), page: schema.maybe(schema.number({ defaultValue: 1, min: 1 })), }), }; +const UninstallTokenMetadataSchema = schema.object({ + id: schema.string(), + policy_id: schema.string(), + policy_name: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])), + created_at: schema.string(), + namespaces: schema.maybe(schema.arrayOf(schema.string())), +}); + +export const GetUninstallTokensMetadataResponseSchema = ListResponseSchema( + UninstallTokenMetadataSchema +); + export const GetUninstallTokenRequestSchema = { params: schema.object({ uninstallTokenId: schema.string(), }), }; + +export const GetUninstallTokenResponseSchema = schema.object({ + item: UninstallTokenMetadataSchema.extends({ + token: schema.string(), + }), +}); diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 1bb954cf990d2..9be09fe4ee554 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -239,6 +239,8 @@ export interface SettingsSOAttributes { fleet_server_hosts?: string[]; secret_storage_requirements_met?: boolean; output_secret_storage_requirements_met?: boolean; + use_space_awareness_migration_status?: 'pending' | 'success' | 'error'; + use_space_awareness_migration_started_at?: string | null; } export interface SpaceSettingsSOAttributes { diff --git a/x-pack/plugins/inference/common/chat_complete/index.ts b/x-pack/plugins/inference/common/chat_complete/index.ts index b42c2217c0177..aef9de12ba7a9 100644 --- a/x-pack/plugins/inference/common/chat_complete/index.ts +++ b/x-pack/plugins/inference/common/chat_complete/index.ts @@ -78,6 +78,8 @@ export type ChatCompletionEvent | ChatCompletionTokenCountEvent | ChatCompletionMessageEvent; +export type FunctionCallingMode = 'native' | 'simulated'; + /** * Request a completion from the LLM based on a prompt or conversation. * @@ -92,5 +94,6 @@ export type ChatCompleteAPI = ( connectorId: string; system?: string; messages: Message[]; + functionCalling?: FunctionCallingMode; } & TToolOptions ) => ChatCompletionResponse; diff --git a/x-pack/plugins/inference/common/chat_complete/request.ts b/x-pack/plugins/inference/common/chat_complete/request.ts index 104d1856c9c80..1038e481a6260 100644 --- a/x-pack/plugins/inference/common/chat_complete/request.ts +++ b/x-pack/plugins/inference/common/chat_complete/request.ts @@ -5,12 +5,13 @@ * 2.0. */ -import type { Message } from '.'; -import { ToolOptions } from './tools'; +import type { Message, FunctionCallingMode } from '.'; +import type { ToolOptions } from './tools'; export type ChatCompleteRequestBody = { connectorId: string; stream?: boolean; system?: string; messages: Message[]; + functionCalling?: FunctionCallingMode; } & ToolOptions; diff --git a/x-pack/plugins/inference/common/output/create_output_api.ts b/x-pack/plugins/inference/common/output/create_output_api.ts index 35fc2b3647004..848135beefb0f 100644 --- a/x-pack/plugins/inference/common/output/create_output_api.ts +++ b/x-pack/plugins/inference/common/output/create_output_api.ts @@ -12,10 +12,11 @@ import { OutputAPI, OutputEvent, OutputEventType } from '.'; import { ensureMultiTurn } from '../ensure_multi_turn'; export function createOutputApi(chatCompleteApi: ChatCompleteAPI): OutputAPI { - return (id, { connectorId, input, schema, system, previousMessages }) => { + return (id, { connectorId, input, schema, system, previousMessages, functionCalling }) => { return chatCompleteApi({ connectorId, system, + functionCalling, messages: ensureMultiTurn([ ...(previousMessages || []), { @@ -26,12 +27,12 @@ export function createOutputApi(chatCompleteApi: ChatCompleteAPI): OutputAPI { ...(schema ? { tools: { - output: { + structuredOutput: { description: `Use the following schema to respond to the user's request in structured data, so it can be parsed and handled.`, schema, }, }, - toolChoice: { function: 'output' as const }, + toolChoice: { function: 'structuredOutput' as const }, } : {}), }).pipe( diff --git a/x-pack/plugins/inference/common/output/index.ts b/x-pack/plugins/inference/common/output/index.ts index d7522f2cfa52e..0f7655f8f1cd4 100644 --- a/x-pack/plugins/inference/common/output/index.ts +++ b/x-pack/plugins/inference/common/output/index.ts @@ -8,7 +8,7 @@ import { Observable } from 'rxjs'; import { ServerSentEventBase } from '@kbn/sse-utils'; import { FromToolSchema, ToolSchema } from '../chat_complete/tool_schema'; -import { Message } from '../chat_complete'; +import type { Message, FunctionCallingMode } from '../chat_complete'; export enum OutputEventType { OutputUpdate = 'output', @@ -61,6 +61,7 @@ export type OutputAPI = < input: string; schema?: TOutputSchema; previousMessages?: Message[]; + functionCalling?: FunctionCallingMode; } ) => Observable< OutputEvent : undefined> diff --git a/x-pack/plugins/inference/public/chat_complete/index.ts b/x-pack/plugins/inference/public/chat_complete/index.ts index 3dfe4616b7323..e229f6c8f8eae 100644 --- a/x-pack/plugins/inference/public/chat_complete/index.ts +++ b/x-pack/plugins/inference/public/chat_complete/index.ts @@ -12,13 +12,14 @@ import type { ChatCompleteRequestBody } from '../../common/chat_complete/request import { httpResponseIntoObservable } from '../util/http_response_into_observable'; export function createChatCompleteApi({ http }: { http: HttpStart }): ChatCompleteAPI { - return ({ connectorId, messages, system, toolChoice, tools }) => { + return ({ connectorId, messages, system, toolChoice, tools, functionCalling }) => { const body: ChatCompleteRequestBody = { connectorId, system, messages, toolChoice, tools, + functionCalling, }; return from( diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts index 62af864a6037d..f1821be4d4d57 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/openai/openai_adapter.ts @@ -13,7 +13,7 @@ import type { ChatCompletionToolMessageParam, ChatCompletionUserMessageParam, } from 'openai/resources'; -import { filter, from, map, switchMap, tap, throwError } from 'rxjs'; +import { filter, from, map, switchMap, tap, throwError, identity } from 'rxjs'; import { Readable, isReadable } from 'stream'; import { ChatCompletionChunkEvent, @@ -26,18 +26,38 @@ import { createTokenLimitReachedError } from '../../../../common/chat_complete/e import { createInferenceInternalError } from '../../../../common/errors'; import { eventSourceStreamIntoObservable } from '../../../util/event_source_stream_into_observable'; import type { InferenceConnectorAdapter } from '../../types'; +import { + wrapWithSimulatedFunctionCalling, + parseInlineFunctionCalls, +} from '../../simulated_function_calling'; export const openAIAdapter: InferenceConnectorAdapter = { - chatComplete: ({ executor, system, messages, toolChoice, tools }) => { + chatComplete: ({ executor, system, messages, toolChoice, tools, functionCalling, logger }) => { const stream = true; + const simulatedFunctionCalling = functionCalling === 'simulated'; - const request: Omit & { model?: string } = { - stream, - messages: messagesToOpenAI({ system, messages }), - tool_choice: toolChoiceToOpenAI(toolChoice), - tools: toolsToOpenAI(tools), - temperature: 0, - }; + let request: Omit & { model?: string }; + if (simulatedFunctionCalling) { + const wrapped = wrapWithSimulatedFunctionCalling({ + system, + messages, + toolChoice, + tools, + }); + request = { + stream, + messages: messagesToOpenAI({ system: wrapped.system, messages: wrapped.messages }), + temperature: 0, + }; + } else { + request = { + stream, + messages: messagesToOpenAI({ system, messages }), + tool_choice: toolChoiceToOpenAI(toolChoice), + tools: toolsToOpenAI(tools), + temperature: 0, + }; + } return from( executor.invoke({ @@ -94,7 +114,8 @@ export const openAIAdapter: InferenceConnectorAdapter = { }; }) ?? [], }; - }) + }), + simulatedFunctionCalling ? parseInlineFunctionCalls({ logger }) : identity ); }, }; diff --git a/x-pack/plugins/inference/server/chat_complete/api.ts b/x-pack/plugins/inference/server/chat_complete/api.ts index fe879392cd4de..ca9e61ff3627f 100644 --- a/x-pack/plugins/inference/server/chat_complete/api.ts +++ b/x-pack/plugins/inference/server/chat_complete/api.ts @@ -31,6 +31,7 @@ export function createChatCompleteApi({ toolChoice, tools, system, + functionCalling, }): ChatCompletionResponse => { return defer(async () => { const actionsClient = await actions.getActionsClientWithRequest(request); @@ -58,6 +59,7 @@ export function createChatCompleteApi({ toolChoice, tools, logger, + functionCalling, }); }), chunksIntoMessage({ diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/constants.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/constants.ts new file mode 100644 index 0000000000000..a25deca07b7d9 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const TOOL_USE_START = '<|tool_use_start|>'; +export const TOOL_USE_END = '<|tool_use_end|>'; diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts new file mode 100644 index 0000000000000..872e842e03f86 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/get_system_instructions.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TOOL_USE_END, TOOL_USE_START } from './constants'; +import { ToolDefinition } from '../../../common/chat_complete/tools'; + +export function getSystemMessageInstructions({ + tools, +}: { + tools?: Record; +}) { + const formattedTools = Object.entries(tools ?? {}).map(([name, tool]) => { + return { + name, + ...tool, + }; + }); + + if (formattedTools.length) { + return `In this environment, you have access to a set of tools you can use to answer the user's question. + + DO NOT call a tool when it is not listed. + ONLY define input that is defined in the tool properties. + If a tool does not have properties, leave them out. + + It is EXTREMELY important that you generate valid JSON between the \`\`\`json and \`\`\` delimiters. + + You may call them like this. + + Given the following tool: + + ${JSON.stringify({ + name: 'my_tool', + description: 'A tool to call', + schema: { + type: 'object', + properties: { + myProperty: { + type: 'string', + }, + }, + }, + })} + + Use it the following way: + + ${TOOL_USE_START} + \`\`\`json + ${JSON.stringify({ name: 'my_tool', input: { myProperty: 'myValue' } })} + \`\`\`\ + ${TOOL_USE_END} + + Given the following tool: + ${JSON.stringify({ + name: 'my_tool_without_parameters', + description: 'A tool to call without parameters', + })} + + Use it the following way: + ${TOOL_USE_START} + \`\`\`json + ${JSON.stringify({ name: 'my_tool_without_parameters', input: {} })} + \`\`\`\ + ${TOOL_USE_END} + + Here are the tools available: + + ${JSON.stringify( + formattedTools.map((tool) => ({ + name: tool.name, + description: tool.description, + ...(tool.schema ? { schema: tool.schema } : {}), + })) + )} + + `; + } + + return `No tools are available anymore. DO NOT UNDER ANY CIRCUMSTANCES call any tool, regardless of whether it was previously called.`; +} diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/index.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/index.ts new file mode 100644 index 0000000000000..8863628f8af68 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { wrapWithSimulatedFunctionCalling } from './wrap_with_simulated_function_calling'; +export { parseInlineFunctionCalls } from './parse_inline_function_calls'; diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts new file mode 100644 index 0000000000000..2fa9dd899e986 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/parse_inline_function_calls.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Observable } from 'rxjs'; +import { Logger } from '@kbn/logging'; +import { + ChatCompletionChunkEvent, + ChatCompletionTokenCountEvent, + ChatCompletionEventType, +} from '../../../common/chat_complete'; +import { createInferenceInternalError } from '../../../common/errors'; +import { TOOL_USE_END, TOOL_USE_START } from './constants'; + +function matchOnSignalStart(buffer: string) { + if (buffer.includes(TOOL_USE_START)) { + const split = buffer.split(TOOL_USE_START); + return [split[0], TOOL_USE_START + split[1]]; + } + + for (let i = 0; i < buffer.length; i++) { + const remaining = buffer.substring(i); + if (TOOL_USE_START.startsWith(remaining)) { + return [buffer.substring(0, i), remaining]; + } + } + + return false; +} + +export function parseInlineFunctionCalls({ logger }: { logger: Logger }) { + return (source: Observable) => { + let functionCallBuffer: string = ''; + + // As soon as we see a TOOL_USE_START token, we write all chunks + // to a buffer, that we flush as a function request if we + // spot the stop sequence. + + return new Observable( + (subscriber) => { + function parseFunctionCall(buffer: string) { + logger.debug('Parsing function call:\n' + buffer); + + const match = buffer.match( + /<\|tool_use_start\|>\s*```json\n?(.*?)(\n```\s*).*<\|tool_use_end\|>/s + ); + + const functionCallBody = match?.[1]; + + if (!functionCallBody) { + throw createInferenceInternalError(`Invalid function call syntax`); + } + + const parsedFunctionCall = JSON.parse(functionCallBody) as { + name?: string; + input?: unknown; + }; + + logger.debug(() => 'Parsed function call:\n ' + JSON.stringify(parsedFunctionCall)); + + if (!parsedFunctionCall.name) { + throw createInferenceInternalError(`Missing name for tool use`); + } + + subscriber.next({ + content: '', + tool_calls: [ + { + index: 0, + toolCallId: parsedFunctionCall.name, + function: { + name: parsedFunctionCall.name, + arguments: JSON.stringify(parsedFunctionCall.input || {}), + }, + }, + ], + type: ChatCompletionEventType.ChatCompletionChunk, + }); + } + + source.subscribe({ + next: (event) => { + if (event.type === ChatCompletionEventType.ChatCompletionTokenCount) { + subscriber.next(event); + return; + } + + const { type, content } = event; + + function next(contentToEmit: string) { + subscriber.next({ + type, + content: contentToEmit, + tool_calls: [], + }); + } + + const match = matchOnSignalStart(functionCallBuffer + content); + + if (match) { + const [beforeStartSignal, afterStartSignal] = match; + functionCallBuffer = afterStartSignal; + if (beforeStartSignal) { + next(beforeStartSignal); + } + + if (functionCallBuffer.includes(TOOL_USE_END)) { + const [beforeEndSignal, afterEndSignal] = functionCallBuffer.split(TOOL_USE_END); + + try { + parseFunctionCall(beforeEndSignal + TOOL_USE_END); + functionCallBuffer = ''; + next(afterEndSignal); + } catch (error) { + subscriber.error(error); + } + } + } else { + functionCallBuffer = ''; + next(content); + } + }, + complete: () => { + subscriber.complete(); + }, + error: (error) => { + subscriber.error(error); + }, + }); + } + ); + }; +} diff --git a/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/wrap_with_simulated_function_calling.ts b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/wrap_with_simulated_function_calling.ts new file mode 100644 index 0000000000000..d8cfc373b66cc --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/simulated_function_calling/wrap_with_simulated_function_calling.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AssistantMessage, Message, ToolMessage, UserMessage } from '../../../common'; +import { MessageRole } from '../../../common/chat_complete'; +import { ToolChoice, ToolChoiceType, ToolDefinition } from '../../../common/chat_complete/tools'; +import { TOOL_USE_END, TOOL_USE_START } from './constants'; +import { getSystemMessageInstructions } from './get_system_instructions'; + +function replaceFunctionsWithTools(content: string) { + return content.replaceAll(/(function)(s|[\s*\.])?(?!\scall)/g, (match, p1, p2) => { + return `tool${p2 || ''}`; + }); +} + +export function wrapWithSimulatedFunctionCalling({ + messages, + system, + tools, + toolChoice, +}: { + messages: Message[]; + system?: string; + tools?: Record; + toolChoice?: ToolChoice; +}): { messages: Message[]; system: string } { + const instructions = getSystemMessageInstructions({ + tools, + }); + + const wrappedSystem = system ? `${system}\n${instructions}` : instructions; + + const wrappedMessages = messages + .map((message) => { + if (message.role === MessageRole.Tool) { + return convertToolResponseMessage(message); + } + if (message.role === MessageRole.Assistant && message.toolCalls?.length) { + return convertToolCallMessage(message); + } + return message; + }) + .map((message) => { + return { + ...message, + content: message.content ? replaceFunctionsWithTools(message.content) : message.content, + }; + }); + + if (toolChoice) { + let selectionMessage; + if (typeof toolChoice === 'object') { + selectionMessage = `Remember, use the ${toolChoice.function} tool to answer this question.`; + } else if (toolChoice === ToolChoiceType.required) { + selectionMessage = `Remember, you MUST use one of the provided tool to answer this question.`; + } else if (toolChoice === ToolChoiceType.auto) { + selectionMessage = `Remember, you CAN use one of the provided tool to answer this question.`; + } + + if (selectionMessage) { + wrappedMessages[messages.length - 1].content += `\n${selectionMessage}`; + } + } + + return { + messages: wrappedMessages as Message[], + system: wrappedSystem, + }; +} + +const convertToolResponseMessage = (message: ToolMessage): UserMessage => { + return { + role: MessageRole.User, + content: JSON.stringify({ + type: 'tool_result', + tool: message.toolCallId, + response: message.response, + }), + }; +}; + +const convertToolCallMessage = (message: AssistantMessage): AssistantMessage => { + // multi-call not supported by simulated mode, there will never be more than one + const toolCall = message.toolCalls![0]; + + let content = message.content || ''; + + content += + TOOL_USE_START + + '\n```json\n' + + JSON.stringify({ + name: toolCall.function.name, + input: 'arguments' in toolCall.function ? toolCall.function.arguments : {}, + }) + + '\n```' + + TOOL_USE_END; + + return { + role: MessageRole.Assistant, + content, + }; +}; diff --git a/x-pack/plugins/inference/server/chat_complete/types.ts b/x-pack/plugins/inference/server/chat_complete/types.ts index 5ef28fdbdc808..394fe370240ef 100644 --- a/x-pack/plugins/inference/server/chat_complete/types.ts +++ b/x-pack/plugins/inference/server/chat_complete/types.ts @@ -10,6 +10,7 @@ import type { Logger } from '@kbn/logging'; import type { ChatCompletionChunkEvent, ChatCompletionTokenCountEvent, + FunctionCallingMode, Message, } from '../../common/chat_complete'; import type { ToolOptions } from '../../common/chat_complete/tools'; @@ -24,9 +25,10 @@ import type { InferenceExecutor } from './utils'; export interface InferenceConnectorAdapter { chatComplete: ( options: { + executor: InferenceExecutor; messages: Message[]; system?: string; - executor: InferenceExecutor; + functionCalling?: FunctionCallingMode; logger: Logger; } & ToolOptions ) => Observable; diff --git a/x-pack/plugins/inference/server/routes/chat_complete.ts b/x-pack/plugins/inference/server/routes/chat_complete.ts index 5a9c0aae50958..fdf33fbf0af82 100644 --- a/x-pack/plugins/inference/server/routes/chat_complete.ts +++ b/x-pack/plugins/inference/server/routes/chat_complete.ts @@ -71,6 +71,9 @@ const chatCompleteBodySchema: Type = schema.object({ }), ]) ), + functionCalling: schema.maybe( + schema.oneOf([schema.literal('native'), schema.literal('simulated')]) + ), }); export function registerChatCompleteRoute({ @@ -96,7 +99,7 @@ export function registerChatCompleteRoute({ const client = createInferenceClient({ request, actions, logger }); - const { connectorId, messages, system, toolChoice, tools } = request.body; + const { connectorId, messages, system, toolChoice, tools, functionCalling } = request.body; const chatCompleteResponse = client.chatComplete({ connectorId, @@ -104,6 +107,7 @@ export function registerChatCompleteRoute({ system, toolChoice, tools, + functionCalling, }); return response.ok({ diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/generate_esql.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/generate_esql.ts index 8a111322a8de6..d31952e2f5252 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/generate_esql.ts +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/generate_esql.ts @@ -21,6 +21,7 @@ import { INLINE_ESQL_QUERY_REGEX } from '../../../../common/tasks/nl_to_esql/con import { EsqlDocumentBase } from '../doc_base'; import { requestDocumentationSchema } from './shared'; import type { NlToEsqlTaskEvent } from '../types'; +import type { FunctionCallingMode } from '../../../../common/chat_complete'; export const generateEsqlTask = ({ chatCompleteApi, @@ -29,6 +30,7 @@ export const generateEsqlTask = ({ messages, toolOptions: { tools, toolChoice }, docBase, + functionCalling, logger, }: { connectorId: string; @@ -37,6 +39,7 @@ export const generateEsqlTask = ({ toolOptions: ToolOptions; chatCompleteApi: InferenceClient['chatComplete']; docBase: EsqlDocumentBase; + functionCalling?: FunctionCallingMode; logger: Pick; }) => { return function askLlmToRespond({ @@ -65,6 +68,7 @@ export const generateEsqlTask = ({ }), chatCompleteApi({ connectorId, + functionCalling, system: `${systemMessage} # Current task diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/request_documentation.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/request_documentation.ts index 05f454c044d31..d4eb3060f59bb 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/request_documentation.ts +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/actions/request_documentation.ts @@ -10,24 +10,28 @@ import { InferenceClient, withoutOutputUpdateEvents } from '../../..'; import { Message } from '../../../../common'; import { ToolChoiceType, ToolOptions } from '../../../../common/chat_complete/tools'; import { requestDocumentationSchema } from './shared'; +import type { FunctionCallingMode } from '../../../../common/chat_complete'; export const requestDocumentation = ({ outputApi, system, messages, connectorId, + functionCalling, toolOptions: { tools, toolChoice }, }: { outputApi: InferenceClient['output']; system: string; messages: Message[]; connectorId: string; + functionCalling?: FunctionCallingMode; toolOptions: ToolOptions; }) => { const hasTools = !isEmpty(tools) && toolChoice !== ToolChoiceType.none; return outputApi('request_documentation', { connectorId, + functionCalling, system, previousMessages: messages, input: `Based on the previous conversation, request documentation diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/doc_base/aliases.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/doc_base/aliases.ts index 29f07af2d1121..6df382a57fd61 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/doc_base/aliases.ts +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/doc_base/aliases.ts @@ -10,7 +10,8 @@ * This is mostly for the case for STATS. */ const aliases: Record = { - STATS: ['STATS_BY', 'BY', 'STATS...BY'], + STATS: ['STATS_BY', 'BY', 'STATS...BY', 'STATS ... BY'], + OPERATORS: ['LIKE', 'RLIKE', 'IN'], }; const getAliasMap = () => { diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt index b9b70ebad625e..ccd7e12517ffb 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/esql_docs/esql-where.txt @@ -21,6 +21,7 @@ WHERE supports the following types of functions: - Type conversation functions - Conditional functions and expressions - Multi-value functions +- Operators Aggregation functions are WHERE supported for EVAL. diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt b/x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt index 2efa08a6288c0..da590d9531ccb 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/system_message.txt @@ -185,7 +185,6 @@ Binary operators: ==, !=, <, <=, >, >=, +, -, *, /, % Logical operators: AND, OR, NOT Predicates: IS NULL, IS NOT NULL Unary operators: - - IN LIKE: filter data based on string patterns using wildcards RLIKE: filter data based on string patterns using regular expressions diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/task.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/task.ts index 04b879351cc54..e0c5a838ea148 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/task.ts +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/task.ts @@ -21,6 +21,7 @@ export function naturalLanguageToEsql({ tools, toolChoice, logger, + functionCalling, ...rest }: NlToEsqlTaskParams): Observable> { return from(loadDocBase()).pipe( @@ -36,6 +37,7 @@ export function naturalLanguageToEsql({ docBase, logger, systemMessage, + functionCalling, toolOptions: { tools, toolChoice, @@ -44,6 +46,7 @@ export function naturalLanguageToEsql({ return requestDocumentation({ connectorId, + functionCalling, outputApi: client.output, messages, system: systemMessage, diff --git a/x-pack/plugins/inference/server/tasks/nl_to_esql/types.ts b/x-pack/plugins/inference/server/tasks/nl_to_esql/types.ts index c460f029b147e..a0bcd635081ea 100644 --- a/x-pack/plugins/inference/server/tasks/nl_to_esql/types.ts +++ b/x-pack/plugins/inference/server/tasks/nl_to_esql/types.ts @@ -9,6 +9,7 @@ import type { Logger } from '@kbn/logging'; import type { ChatCompletionChunkEvent, ChatCompletionMessageEvent, + FunctionCallingMode, Message, } from '../../../common/chat_complete'; import type { ToolOptions } from '../../../common/chat_complete/tools'; @@ -27,5 +28,6 @@ export type NlToEsqlTaskParams = { client: Pick; connectorId: string; logger: Pick; + functionCalling?: FunctionCallingMode; } & TToolOptions & ({ input: string } | { messages: Message[] }); diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/build_integration.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/build_integration.ts index 3161f06f8a6ae..81f2231bcb05d 100644 --- a/x-pack/plugins/integration_assistant/__jest__/fixtures/build_integration.ts +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/build_integration.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { Integration } from '../../common/api/model/common_attributes'; +import { Integration } from '../../common'; export const testIntegration: Integration = { name: 'integration', diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts index 5f68dfc4668ec..80366e7bd6f93 100644 --- a/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SamplesFormatName } from '../../common/api/model/common_attributes'; +import { SamplesFormatName } from '../../common'; import type { Pipeline } from '../../common'; export const categorizationInitialPipeline: Pipeline = { diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts index eba49baaa8595..67e6c7d8f6a5e 100644 --- a/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SamplesFormatName } from '../../common/api/model/common_attributes'; +import { SamplesFormatName } from '../../common'; export const ecsMappingExpectedResults = { mapping: { diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts index 587d8987c0960..49c8d2e16809b 100644 --- a/x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SamplesFormatName } from '../../common/api/model/common_attributes'; +import { SamplesFormatName } from '../../common'; export const kvState = { lastExecutedChain: 'testchain', diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts index 31d7208eb4d87..b3c1e4c05ebd9 100644 --- a/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SamplesFormatName } from '../../common/api/model/common_attributes'; +import { SamplesFormatName } from '../../common'; export const logFormatDetectionTestState = { lastExecutedChain: 'testchain', diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts index 9d0915735d52a..d96d845ae43b6 100644 --- a/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SamplesFormatName } from '../../common/api/model/common_attributes'; +import { SamplesFormatName } from '../../common'; import type { Pipeline } from '../../common'; export const relatedInitialPipeline: Pipeline = { diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.gen.ts similarity index 95% rename from x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts rename to x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.gen.ts index 597c44fde54a5..a224bb3cbe241 100644 --- a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts +++ b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.gen.ts @@ -17,13 +17,13 @@ import { z } from '@kbn/zod'; import { + PackageName, + DataStreamName, LogSamples, Connector, LangSmithOptions, - DataStreamName, - PackageName, -} from '../model/common_attributes'; -import { AnalyzeLogsAPIResponse } from '../model/response_schemas'; +} from '../model/common_attributes.gen'; +import { AnalyzeLogsAPIResponse } from '../model/response_schemas.gen'; export type AnalyzeLogsRequestBody = z.infer; export const AnalyzeLogsRequestBody = z.object({ diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml index 23050bc6a50fc..165a2cff91a06 100644 --- a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml @@ -7,7 +7,7 @@ paths: post: summary: Analyzes log samples and processes them. operationId: AnalyzeLogs - x-codegen-enabled: false + x-codegen-enabled: true description: Analyzes log samples and processes them tags: - Analyze Logs API diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts index 08595c1ce1911..30848ca8945cf 100644 --- a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts +++ b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts @@ -6,7 +6,7 @@ */ import { expectParseSuccess } from '@kbn/zod-helpers'; -import { AnalyzeLogsRequestBody } from './analyze_logs_route'; +import { AnalyzeLogsRequestBody } from './analyze_logs_route.gen'; import { getAnalyzeLogsRequestBody } from '../model/api_test.mock'; describe('Analyze Logs request schema', () => { diff --git a/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.ts b/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.gen.ts similarity index 64% rename from x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.ts rename to x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.gen.ts index ce9c919c0d43d..46db1066ffdff 100644 --- a/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.ts +++ b/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.gen.ts @@ -5,9 +5,18 @@ * 2.0. */ +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Integration Assistatnt Build Integrarion API endpoint + * version: 1 + */ + import { z } from '@kbn/zod'; -import { Integration } from '../model/common_attributes'; +import { Integration } from '../model/common_attributes.gen'; export type BuildIntegrationRequestBody = z.infer; export const BuildIntegrationRequestBody = z.object({ diff --git a/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.schema.yaml b/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.schema.yaml index 798ead34114a6..7a5a994cff4df 100644 --- a/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/build_integration/build_integration.schema.yaml @@ -7,7 +7,7 @@ paths: post: summary: Builds Integration with the given input samples operationId: BuildIntegration - x-codegen-enabled: false + x-codegen-enabled: true description: Build Integration for the given input samples tags: - Build Integration API diff --git a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.gen.ts similarity index 78% rename from x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts rename to x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.gen.ts index c78aa745671e9..0b2492e41ac12 100644 --- a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts +++ b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.gen.ts @@ -5,27 +5,36 @@ * 2.0. */ +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Integration Assistatnt Categorization API endpoint + * version: 1 + */ + import { z } from '@kbn/zod'; import { - Connector, - DataStreamName, - LangSmithOptions, PackageName, - Pipeline, + DataStreamName, RawSamples, + Pipeline, + Connector, SamplesFormat, -} from '../model/common_attributes'; -import { CategorizationAPIResponse } from '../model/response_schemas'; + LangSmithOptions, +} from '../model/common_attributes.gen'; +import { CategorizationAPIResponse } from '../model/response_schemas.gen'; export type CategorizationRequestBody = z.infer; export const CategorizationRequestBody = z.object({ packageName: PackageName, dataStreamName: DataStreamName, rawSamples: RawSamples, - samplesFormat: SamplesFormat, currentPipeline: Pipeline, connectorId: Connector, + samplesFormat: SamplesFormat, langSmithOptions: LangSmithOptions.optional(), }); export type CategorizationRequestBodyInput = z.input; diff --git a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml index e548f4d816776..4a8a28abf3a7e 100644 --- a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml @@ -7,7 +7,7 @@ paths: post: summary: Builds Categorization processors based on the samples operationId: Categorization - x-codegen-enabled: false + x-codegen-enabled: true description: Perform Categorization for the given ecs mappings. tags: - Categorization API diff --git a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.test.ts b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.test.ts index f7ef31f5fdb99..fc7f6f9736d36 100644 --- a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.test.ts +++ b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.test.ts @@ -6,7 +6,7 @@ */ import { expectParseSuccess } from '@kbn/zod-helpers'; -import { CategorizationRequestBody } from './categorization_route'; +import { CategorizationRequestBody } from './categorization_route.gen'; import { getCategorizationRequestMock } from '../model/api_test.mock'; describe('Categorization request schema', () => { diff --git a/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.ts b/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.gen.ts similarity index 70% rename from x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.ts rename to x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.gen.ts index e2a9255490f5e..18888108396f4 100644 --- a/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.ts +++ b/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.gen.ts @@ -5,10 +5,19 @@ * 2.0. */ +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Integration Assistatnt Check Pipeline API endpoint + * version: 1 + */ + import { z } from '@kbn/zod'; -import { Pipeline, RawSamples } from '../model/common_attributes'; -import { CheckPipelineAPIResponse } from '../model/response_schemas'; +import { RawSamples, Pipeline } from '../model/common_attributes.gen'; +import { CheckPipelineAPIResponse } from '../model/response_schemas.gen'; export type CheckPipelineRequestBody = z.infer; export const CheckPipelineRequestBody = z.object({ diff --git a/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.schema.yaml b/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.schema.yaml index 22785fc40bbf2..b7f20616007ea 100644 --- a/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/check_pipeline/check_pipeline.schema.yaml @@ -7,7 +7,7 @@ paths: post: summary: Checks if the pipeline is valid for the given samples operationId: CheckPipeline - x-codegen-enabled: false + x-codegen-enabled: true description: Check latest pipeline against the input samples. tags: - Check Pipeline API diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.gen.ts similarity index 91% rename from x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts rename to x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.gen.ts index 58143ec7177d7..867f792405cbd 100644 --- a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts +++ b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.gen.ts @@ -20,13 +20,13 @@ import { PackageName, DataStreamName, RawSamples, + SamplesFormat, Mapping, Connector, LangSmithOptions, - SamplesFormat, -} from '../model/common_attributes'; -import { ESProcessorItem } from '../model/processor_attributes'; -import { EcsMappingAPIResponse } from '../model/response_schemas'; +} from '../model/common_attributes.gen'; +import { ESProcessorItem } from '../model/processor_attributes.gen'; +import { EcsMappingAPIResponse } from '../model/response_schemas.gen'; export type EcsMappingRequestBody = z.infer; export const EcsMappingRequestBody = z.object({ diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml index 7026fc6d86f96..6bc125a74f52a 100644 --- a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml @@ -7,7 +7,7 @@ paths: post: summary: Builds ECS Mapping based on the input samples operationId: EcsMapping - x-codegen-enabled: false + x-codegen-enabled: true description: Perform ECS mapping for the given input JSON samples tags: - ECS Mapping API diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.test.ts b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.test.ts index 770c3ff96f675..de4820633b825 100644 --- a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.test.ts +++ b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.test.ts @@ -6,7 +6,7 @@ */ import { expectParseSuccess } from '@kbn/zod-helpers'; -import { EcsMappingRequestBody } from './ecs_route'; +import { EcsMappingRequestBody } from './ecs_route.gen'; import { getEcsMappingRequestMock } from '../model/api_test.mock'; describe('Ecs Mapping request schema', () => { diff --git a/x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts b/x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts index 34ce49ae5b776..de29624cbb21a 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts +++ b/x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts @@ -5,12 +5,12 @@ * 2.0. */ -import type { AnalyzeLogsRequestBody } from '../analyze_logs/analyze_logs_route'; -import type { BuildIntegrationRequestBody } from '../build_integration/build_integration'; -import type { CategorizationRequestBody } from '../categorization/categorization_route'; -import type { EcsMappingRequestBody } from '../ecs/ecs_route'; -import type { RelatedRequestBody } from '../related/related_route'; -import type { DataStream, Integration, Pipeline } from './common_attributes'; +import type { AnalyzeLogsRequestBody } from '../analyze_logs/analyze_logs_route.gen'; +import type { BuildIntegrationRequestBody } from '../build_integration/build_integration.gen'; +import type { CategorizationRequestBody } from '../categorization/categorization_route.gen'; +import type { EcsMappingRequestBody } from '../ecs/ecs_route.gen'; +import type { RelatedRequestBody } from '../related/related_route.gen'; +import type { DataStream, Integration, Pipeline } from './common_attributes.gen'; const rawSamples = ['{"test1": "test1"}']; diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts similarity index 94% rename from x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts rename to x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts index 964aa9386a87f..59fe7d461698f 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts +++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts @@ -5,9 +5,18 @@ * 2.0. */ +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Common Rule Attributes + * version: not applicable + */ + import { z } from '@kbn/zod'; -import { ESProcessorItem } from './processor_attributes'; +import { ESProcessorItem } from './processor_attributes.gen'; /** * Package name for the integration to be built. diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml index 4c4810b22f9be..0af01834970c7 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml @@ -4,7 +4,7 @@ info: version: "not applicable" paths: {} components: - x-codegen-enabled: false + x-codegen-enabled: true schemas: PackageName: type: string diff --git a/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.gen.ts b/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.gen.ts new file mode 100644 index 0000000000000..5916bcac4d839 --- /dev/null +++ b/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.gen.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Common Rule Attributes + * version: not applicable + */ + +import type { ZodTypeDef } from '@kbn/zod'; +import { z } from '@kbn/zod'; + +/** + * Processor options for the Elasticsearch processor. + */ +export interface ESProcessorOptions { + /** + * An array of items to execute if the processor fails. + */ + on_failure?: ESProcessorItem[]; + /** + * If true, the processor continues to the next processor if the current processor fails. + */ + ignore_failure?: boolean; + /** + * If true, the processor continues to the next processor if the field is missing. + */ + ignore_missing?: boolean; + /** + * Conditionally execute the processor. + */ + if?: string; + /** + * A tag to assign to the document after processing. + */ + tag?: string; + [key: string]: unknown; +} +export interface ESProcessorOptionsInput { + /** + * An array of items to execute if the processor fails. + */ + on_failure?: ESProcessorItemInput[]; + /** + * If true, the processor continues to the next processor if the current processor fails. + */ + ignore_failure?: boolean; + /** + * If true, the processor continues to the next processor if the field is missing. + */ + ignore_missing?: boolean; + /** + * Conditionally execute the processor. + */ + if?: string; + /** + * A tag to assign to the document after processing. + */ + tag?: string; + [key: string]: unknown; +} +export const ESProcessorOptions: z.ZodType< + ESProcessorOptions, + ZodTypeDef, + ESProcessorOptionsInput +> = z + .object({ + /** + * An array of items to execute if the processor fails. + */ + on_failure: z.array(z.lazy(() => ESProcessorItem)).optional(), + /** + * If true, the processor continues to the next processor if the current processor fails. + */ + ignore_failure: z.boolean().optional(), + /** + * If true, the processor continues to the next processor if the field is missing. + */ + ignore_missing: z.boolean().optional(), + /** + * Conditionally execute the processor. + */ + if: z.string().optional(), + /** + * A tag to assign to the document after processing. + */ + tag: z.string().optional(), + }) + .catchall(z.unknown()); + +/** + * Processor item for the Elasticsearch processor. + */ +export interface ESProcessorItem { + [key: string]: ESProcessorOptions; +} +export interface ESProcessorItemInput { + [key: string]: ESProcessorOptionsInput; +} +export const ESProcessorItem: z.ZodType = z + .object({}) + .catchall(z.lazy(() => ESProcessorOptions)); diff --git a/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.schema.yaml index a0c930361737a..837ff5525df88 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.schema.yaml @@ -4,7 +4,7 @@ info: version: "not applicable" paths: {} components: - x-codegen-enabled: false + x-codegen-enabled: true schemas: ESProcessorItem: type: object diff --git a/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.ts b/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.ts deleted file mode 100644 index 4bbebb6b4686c..0000000000000 --- a/x-pack/plugins/integration_assistant/common/api/model/processor_attributes.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; - -/** - * Processor item for the Elasticsearch processor. - */ -export type ESProcessorItem = Record; -export const ESProcessorItem: z.ZodType = z - .object({}) - .catchall(z.lazy(() => ESProcessorOptions)); - -/** - * Processor options for the Elasticsearch processor. - */ -export interface ESProcessorOptions { - on_failure?: ESProcessorItem[]; - ignore_failure?: boolean; - ignore_missing?: boolean; - if?: string; - tag?: string; - [key: string]: unknown; -} -export const ESProcessorOptions = z - .object({ - /** - * An array of items to execute if the processor fails. - */ - on_failure: z.array(ESProcessorItem).optional(), - /** - * If true, the processor continues to the next processor if the current processor fails. - */ - ignore_failure: z.boolean().optional(), - /** - * If true, the processor continues to the next processor if the field is missing. - */ - ignore_missing: z.boolean().optional(), - /** - * Conditionally execute the processor. - */ - if: z.string().optional(), - /** - * A tag to assign to the document after processing. - */ - tag: z.string().optional(), - }) - .catchall(z.unknown()); diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.gen.ts similarity index 91% rename from x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts rename to x-pack/plugins/integration_assistant/common/api/model/response_schemas.gen.ts index c496aa8493723..acb4954c21b90 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts +++ b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.gen.ts @@ -16,8 +16,8 @@ import { z } from '@kbn/zod'; -import { Docs, Mapping, Pipeline, SamplesFormat } from './common_attributes'; -import { ESProcessorItem } from './processor_attributes'; +import { Mapping, Pipeline, Docs, SamplesFormat } from './common_attributes.gen'; +import { ESProcessorItem } from './processor_attributes.gen'; export type EcsMappingAPIResponse = z.infer; export const EcsMappingAPIResponse = z.object({ @@ -52,9 +52,9 @@ export const CheckPipelineAPIResponse = z.object({ export type AnalyzeLogsAPIResponse = z.infer; export const AnalyzeLogsAPIResponse = z.object({ + additionalProcessors: z.array(ESProcessorItem).optional(), results: z.object({ samplesFormat: SamplesFormat, parsedSamples: z.array(z.string()), }), - additionalProcessors: z.array(ESProcessorItem).optional(), }); diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml index 72f853822b09e..c504ad8b17d16 100644 --- a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml @@ -4,7 +4,7 @@ info: version: "not applicable" paths: {} components: - x-codegen-enabled: false + x-codegen-enabled: true schemas: EcsMappingAPIResponse: type: object diff --git a/x-pack/plugins/integration_assistant/common/api/related/related_route.ts b/x-pack/plugins/integration_assistant/common/api/related/related_route.gen.ts similarity index 73% rename from x-pack/plugins/integration_assistant/common/api/related/related_route.ts rename to x-pack/plugins/integration_assistant/common/api/related/related_route.gen.ts index 14796dc5350ba..eceb01679a442 100644 --- a/x-pack/plugins/integration_assistant/common/api/related/related_route.ts +++ b/x-pack/plugins/integration_assistant/common/api/related/related_route.gen.ts @@ -5,27 +5,36 @@ * 2.0. */ +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Integration Assistatnt Related API endpoint + * version: 1 + */ + import { z } from '@kbn/zod'; import { - Connector, - DataStreamName, - LangSmithOptions, PackageName, - Pipeline, + DataStreamName, RawSamples, + Pipeline, + Connector, SamplesFormat, -} from '../model/common_attributes'; -import { RelatedAPIResponse } from '../model/response_schemas'; + LangSmithOptions, +} from '../model/common_attributes.gen'; +import { RelatedAPIResponse } from '../model/response_schemas.gen'; export type RelatedRequestBody = z.infer; export const RelatedRequestBody = z.object({ packageName: PackageName, dataStreamName: DataStreamName, rawSamples: RawSamples, - samplesFormat: SamplesFormat, currentPipeline: Pipeline, connectorId: Connector, + samplesFormat: SamplesFormat, langSmithOptions: LangSmithOptions.optional(), }); export type RelatedRequestBodyInput = z.input; diff --git a/x-pack/plugins/integration_assistant/common/api/related/related_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/related/related_route.schema.yaml index 71fee7e616709..9ca2f6ab7bae0 100644 --- a/x-pack/plugins/integration_assistant/common/api/related/related_route.schema.yaml +++ b/x-pack/plugins/integration_assistant/common/api/related/related_route.schema.yaml @@ -7,7 +7,7 @@ paths: post: summary: Builds related.* fields for integration with the given input samples operationId: Related - x-codegen-enabled: false + x-codegen-enabled: true description: Add Related mappings for the given samples. tags: - Related API diff --git a/x-pack/plugins/integration_assistant/common/api/related/related_route.test.ts b/x-pack/plugins/integration_assistant/common/api/related/related_route.test.ts index 8f69c13303056..fd6a81a61a1ad 100644 --- a/x-pack/plugins/integration_assistant/common/api/related/related_route.test.ts +++ b/x-pack/plugins/integration_assistant/common/api/related/related_route.test.ts @@ -6,7 +6,7 @@ */ import { expectParseSuccess } from '@kbn/zod-helpers'; -import { RelatedRequestBody } from './related_route'; +import { RelatedRequestBody } from './related_route.gen'; import { getRelatedRequestMock } from '../model/api_test.mock'; describe('Related request schema', () => { diff --git a/x-pack/plugins/integration_assistant/common/index.ts b/x-pack/plugins/integration_assistant/common/index.ts index 5310fa67c8cac..21ee814655e10 100644 --- a/x-pack/plugins/integration_assistant/common/index.ts +++ b/x-pack/plugins/integration_assistant/common/index.ts @@ -4,18 +4,21 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -export { BuildIntegrationRequestBody } from './api/build_integration/build_integration'; +export { BuildIntegrationRequestBody } from './api/build_integration/build_integration.gen'; export { CategorizationRequestBody, CategorizationResponse, -} from './api/categorization/categorization_route'; +} from './api/categorization/categorization_route.gen'; export { CheckPipelineRequestBody, CheckPipelineResponse, -} from './api/check_pipeline/check_pipeline'; -export { EcsMappingRequestBody, EcsMappingResponse } from './api/ecs/ecs_route'; -export { RelatedRequestBody, RelatedResponse } from './api/related/related_route'; -export { AnalyzeLogsRequestBody, AnalyzeLogsResponse } from './api/analyze_logs/analyze_logs_route'; +} from './api/check_pipeline/check_pipeline.gen'; +export { EcsMappingRequestBody, EcsMappingResponse } from './api/ecs/ecs_route.gen'; +export { RelatedRequestBody, RelatedResponse } from './api/related/related_route.gen'; +export { + AnalyzeLogsRequestBody, + AnalyzeLogsResponse, +} from './api/analyze_logs/analyze_logs_route.gen'; export type { DataStream, @@ -24,8 +27,10 @@ export type { Pipeline, Docs, SamplesFormat, -} from './api/model/common_attributes'; -export type { ESProcessorItem } from './api/model/processor_attributes'; + LangSmithOptions, +} from './api/model/common_attributes.gen'; +export { SamplesFormatName } from './api/model/common_attributes.gen'; +export type { ESProcessorItem } from './api/model/processor_attributes.gen'; export { CATEGORIZATION_GRAPH_PATH, diff --git a/x-pack/plugins/integration_assistant/public/common/lib/lang_smith.ts b/x-pack/plugins/integration_assistant/public/common/lib/lang_smith.ts index 7234870439930..a8c4f258e20da 100644 --- a/x-pack/plugins/integration_assistant/public/common/lib/lang_smith.ts +++ b/x-pack/plugins/integration_assistant/public/common/lib/lang_smith.ts @@ -11,7 +11,7 @@ import { } from '@kbn/elastic-assistant/impl/assistant_context/constants'; import { Storage } from '@kbn/kibana-utils-plugin/public'; import type { TraceOptions } from '@kbn/elastic-assistant/impl/assistant/types'; -import type { LangSmithOptions } from '../../../common/api/model/common_attributes'; +import type { LangSmithOptions } from '../../../common'; const sessionStorage = new Storage(window.sessionStorage); diff --git a/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts b/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts index 242f6955700e8..e2d91c094d520 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts @@ -8,10 +8,9 @@ import { safeLoad } from 'js-yaml'; import { Environment, FileSystemLoader } from 'nunjucks'; import { join as joinPath } from 'path'; -import { Pipeline } from '../../../common/api/model/common_attributes'; +import { Pipeline, ESProcessorItem } from '../../../common'; import type { EcsMappingState } from '../../types'; import { ECS_TYPES } from './constants'; -import { ESProcessorItem } from '../../../common/api/model/processor_attributes'; import { deepCopy } from '../../util/util'; interface IngestPipeline { diff --git a/x-pack/plugins/integration_assistant/server/util/samples.test.ts b/x-pack/plugins/integration_assistant/server/util/samples.test.ts new file mode 100644 index 0000000000000..131135e842334 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/util/samples.test.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from './samples'; + +describe('merge', () => { + it('Should return source if target is empty', async () => { + const target = {}; + const source = { target: 'user.name', confidence: 0.9, type: 'string', date_formats: [] }; + + const result = merge(target, source); + + expect(result).toEqual(source); + }); + + it('Should return target if source is empty', async () => { + const target = { hostname: '0.0.0.0', 'teleport.internal/resource-id': '1234' }; + const source = {}; + + const result = merge(target, source); + + expect(result).toEqual(target); + }); + + it('Should return one result', async () => { + const target = { + aaa: { + ei: 0, + event: 'cert.create', + uid: '1234', + cluster_name: 'cluster.com', + identity: { user: 'teleport-admin' }, + server_labels: { hostname: 'some-hostname' }, + }, + }; + const source = { + aaa: { + ei: 0, + event: 'session.start', + uid: '4567', + cluster_name: 'cluster.com', + user: 'teleport-admin', + server_labels: { hostname: 'some-other-hostname', id: '1234' }, + }, + }; + + const result = merge(target, source); + + expect(result).toEqual({ + aaa: { + ei: 0, + event: 'cert.create', + uid: '1234', + cluster_name: 'cluster.com', + identity: { user: 'teleport-admin' }, + server_labels: { hostname: 'some-hostname', id: '1234' }, + user: 'teleport-admin', + }, + }); + }); + + it('Should not merge built-in properties of neither target nor source', async () => { + const target = { + __proto__: 'some properties', + constructor: 'some other properties', + hostname: '0.0.0.0', + 'teleport.internal/resource-id': '1234', + }; + const source = { + __proto__: 'some properties of source', + constructor: 'some other properties of source', + }; + + const result = merge(target, source); + + expect(result).toEqual({ hostname: '0.0.0.0', 'teleport.internal/resource-id': '1234' }); + }); + + it('Should keep source object if it collides with target key that is not an object', async () => { + const target = { + hostname: '', + 'teleport.internal/resource-id': '1234', + date_formats: 'format', + }; + const source = { + target: 'user.name', + confidence: 0.9, + type: 'string', + date_formats: { key: 'value' }, + }; + + const result = merge(target, source); + + expect(result).toEqual({ + 'teleport.internal/resource-id': '1234', + target: 'user.name', + confidence: 0.9, + type: 'string', + hostname: '', + date_formats: { key: 'value' }, + }); + }); + + it('Should copy array into the result', async () => { + const target = { date_formats: ['a', 'b'] }; + const source = { target: 'user.name', confidence: 0.9, type: 'string', date_formats: ['c'] }; + + const result = merge(target, source); + + expect(result).toEqual(source); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/util/samples.ts b/x-pack/plugins/integration_assistant/server/util/samples.ts index 766856f644a86..745c3d214095f 100644 --- a/x-pack/plugins/integration_assistant/server/util/samples.ts +++ b/x-pack/plugins/integration_assistant/server/util/samples.ts @@ -163,39 +163,61 @@ export function generateFields(mergedDocs: string): string { return yaml.safeDump(fieldsStructure, { sortKeys: false }); } -function isEmptyValue(value: unknown): boolean { - return ( - value === null || - value === undefined || - (typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length === 0) || - (Array.isArray(value) && value.length === 0) - ); -} - export function merge( target: Record, source: Record ): Record { + const filteredTarget = filterOwnProperties(target); for (const [key, sourceValue] of Object.entries(source)) { - const targetValue = target[key]; - if (Array.isArray(sourceValue)) { - // Directly assign arrays - target[key] = sourceValue; - } else if ( - typeof sourceValue === 'object' && - sourceValue !== null && - !Array.isArray(targetValue) - ) { - if (typeof targetValue !== 'object' || isEmptyValue(targetValue)) { - target[key] = merge({}, sourceValue); - } else { - target[key] = merge(targetValue, sourceValue); + if (!isBuiltInProperties(key, source)) { + const targetValue = filteredTarget[key]; + if (Array.isArray(sourceValue)) { + // Directly assign arrays + filteredTarget[key] = sourceValue; + } else if (isObject(sourceValue) && !Array.isArray(targetValue)) { + if (!isObject(targetValue) || isEmptyValue(targetValue)) { + filteredTarget[key] = merge({}, sourceValue); + } else { + filteredTarget[key] = merge(targetValue, sourceValue); + } + } else if ( + !(key in filteredTarget) || + (isEmptyValue(targetValue) && !isEmptyValue(sourceValue)) + ) { + filteredTarget[key] = sourceValue; } - } else if (!(key in target) || (isEmptyValue(targetValue) && !isEmptyValue(sourceValue))) { - target[key] = sourceValue; } } - return target; + return filteredTarget; +} + +function isEmptyValue(value: unknown): boolean { + if (value == null) return true; + if (isObject(value)) { + if (Array.isArray(value)) return value.length === 0; + return value && Object.keys(value).length === 0; + } + return false; +} + +function isObject(value: any): boolean { + return typeof value === 'object' && value !== null; +} + +function isBuiltInProperties(key: string, obj: Record): boolean { + return key === 'constructor' || !Object.prototype.hasOwnProperty.call(obj, key); +} + +function filterOwnProperties(obj: Record): Record { + const ownProps: Record = {}; + + for (const key of Object.getOwnPropertyNames(obj)) { + if (!isBuiltInProperties(key, obj)) { + ownProps[key] = (obj as any)[key]; + } + } + + return ownProps; } export function mergeSamples(objects: any[]): string { diff --git a/x-pack/plugins/ml/public/cases/register_cases_attachments.ts b/x-pack/plugins/ml/public/cases/register_cases_attachments.ts index 2625affdec331..9263af617f69a 100644 --- a/x-pack/plugins/ml/public/cases/register_cases_attachments.ts +++ b/x-pack/plugins/ml/public/cases/register_cases_attachments.ts @@ -10,21 +10,14 @@ import type { CoreStart } from '@kbn/core/public'; import { registerAnomalyChartsCasesAttachment } from './register_anomaly_charts_attachment'; import { registerSingleMetricViewerCasesAttachment } from './register_single_metric_viewer_attachment'; import type { MlStartDependencies } from '../plugin'; -import type { SingleMetricViewerServices } from '../embeddables/types'; import { registerAnomalySwimLaneCasesAttachment } from './register_anomaly_swim_lane_attachment'; export function registerCasesAttachments( cases: CasesPublicSetup, coreStart: CoreStart, - pluginStart: MlStartDependencies, - singleMetricViewerServices: SingleMetricViewerServices + pluginStart: MlStartDependencies ) { registerAnomalySwimLaneCasesAttachment(cases, pluginStart); registerAnomalyChartsCasesAttachment(cases, coreStart, pluginStart); - registerSingleMetricViewerCasesAttachment( - cases, - coreStart, - pluginStart, - singleMetricViewerServices - ); + registerSingleMetricViewerCasesAttachment(cases, coreStart, pluginStart); } diff --git a/x-pack/plugins/ml/public/cases/register_single_metric_viewer_attachment.tsx b/x-pack/plugins/ml/public/cases/register_single_metric_viewer_attachment.tsx index 8ef4c0bc3813c..c969aeafbabf0 100644 --- a/x-pack/plugins/ml/public/cases/register_single_metric_viewer_attachment.tsx +++ b/x-pack/plugins/ml/public/cases/register_single_metric_viewer_attachment.tsx @@ -14,21 +14,14 @@ import { PLUGIN_ICON } from '../../common/constants/app'; import { CASE_ATTACHMENT_TYPE_ID_SINGLE_METRIC_VIEWER } from '../../common/constants/cases'; import type { MlStartDependencies } from '../plugin'; import { getSingleMetricViewerComponent } from '../shared_components/single_metric_viewer'; -import type { SingleMetricViewerServices } from '../embeddables/types'; import type { MlDependencies } from '../application/app'; +import { getMlServices } from '../embeddables/single_metric_viewer/get_services'; export function registerSingleMetricViewerCasesAttachment( cases: CasesPublicSetup, coreStart: CoreStart, - pluginStart: MlStartDependencies, - mlServices: SingleMetricViewerServices + pluginStart: MlStartDependencies ) { - const SingleMetricViewerComponent = getSingleMetricViewerComponent( - coreStart, - pluginStart as MlDependencies, - mlServices - ); - cases.attachmentFramework.registerPersistableState({ id: CASE_ATTACHMENT_TYPE_ID_SINGLE_METRIC_VIEWER, icon: PLUGIN_ICON, @@ -44,7 +37,15 @@ export function registerSingleMetricViewerCasesAttachment( ), timelineAvatar: PLUGIN_ICON, children: React.lazy(async () => { - const { initComponent } = await import('./single_metric_viewer_attachment'); + const [{ initComponent }, mlServices] = await Promise.all([ + import('./single_metric_viewer_attachment'), + getMlServices(coreStart, pluginStart), + ]); + const SingleMetricViewerComponent = getSingleMetricViewerComponent( + coreStart, + pluginStart as MlDependencies, + mlServices + ); return { default: initComponent(pluginStart.fieldFormats, SingleMetricViewerComponent), }; diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts index 7246f60337dd9..be6e0c3305230 100644 --- a/x-pack/plugins/ml/public/plugin.ts +++ b/x-pack/plugins/ml/public/plugin.ts @@ -73,7 +73,6 @@ import type { ElasticModels } from './application/services/elastic_models_servic import type { MlApi } from './application/services/ml_api_service'; import type { MlCapabilities } from '../common/types/capabilities'; import { AnomalySwimLane } from './shared_components'; -import { getMlServices } from './embeddables/single_metric_viewer/get_services'; export interface MlStartDependencies { cases?: CasesPublicStart; @@ -275,8 +274,7 @@ export class MlPlugin implements Plugin { registerEmbeddables(pluginsSetup.embeddable, core); if (pluginsSetup.cases) { - const mlServices = await getMlServices(coreStart, pluginStart); - registerCasesAttachments(pluginsSetup.cases, coreStart, pluginStart, mlServices); + registerCasesAttachments(pluginsSetup.cases, coreStart, pluginStart); } if (pluginsSetup.maps) { diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx new file mode 100644 index 0000000000000..4054614838954 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/entity_link.test.tsx @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { render, screen } from '@testing-library/react'; +import { createMemoryHistory } from 'history'; +import React from 'react'; +import { EntityLink } from '.'; +import { MockApmPluginContextWrapper } from '../../../../context/apm_plugin/mock_apm_plugin_context'; +import type { ServiceEntitySummary } from '../../../../context/apm_service/use_service_entity_summary_fetcher'; +import * as useServiceEntitySummary from '../../../../context/apm_service/use_service_entity_summary_fetcher'; +import type { EntityManagerEnablementContextValue } from '../../../../context/entity_manager_context/entity_manager_context'; +import * as useEntityManagerEnablementContext from '../../../../context/entity_manager_context/use_entity_manager_enablement_context'; +import * as useFetcher from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; +import { fromQuery } from '../../../shared/links/url_helpers'; +import { APIReturnType } from '../../../../services/rest/create_call_apm_api'; +import { Redirect } from 'react-router-dom'; +import { ApmPluginContextValue } from '../../../../context/apm_plugin/apm_plugin_context'; +import { ApmThemeProvider } from '../../../routing/app_root'; + +jest.mock('react-router-dom', () => ({ + ...jest.requireActual('react-router-dom'), // Keep other functionality intact + Redirect: jest.fn(() =>
Mocked Redirect
), // Mock Redirect with a custom implementation +})); + +export type HasApmData = APIReturnType<'GET /internal/apm/has_data'>; + +const renderEntityLink = ({ + entityManagerMockReturnValue, + serviceEntitySummaryMockReturnValue, + hasApmDataFetcherMockReturnValue, + query = {}, +}: { + entityManagerMockReturnValue: Partial; + serviceEntitySummaryMockReturnValue: ReturnType< + typeof useServiceEntitySummary.useServiceEntitySummaryFetcher + >; + hasApmDataFetcherMockReturnValue: { data?: HasApmData; status: FETCH_STATUS }; + query?: { + rangeFrom?: string; + rangeTo?: string; + }; +}) => { + jest + .spyOn(useEntityManagerEnablementContext, 'useEntityManagerEnablementContext') + .mockReturnValue( + entityManagerMockReturnValue as unknown as EntityManagerEnablementContextValue + ); + + jest + .spyOn(useServiceEntitySummary, 'useServiceEntitySummaryFetcher') + .mockReturnValue(serviceEntitySummaryMockReturnValue); + + jest.spyOn(useFetcher, 'useFetcher').mockReturnValue({ + ...hasApmDataFetcherMockReturnValue, + refetch: jest.fn(), + }); + + const history = createMemoryHistory(); + + history.replace({ + pathname: '/link-to/entity/foo', + search: fromQuery(query), + }); + + const { rerender, ...tools } = render( + ({ + from: 'now-24h', + to: 'now', + }), + }, + }, + }, + }, + }, + } as unknown as ApmPluginContextValue + } + > + + + + + ); + return { rerender, ...tools }; +}; + +describe('Entity link', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('renders a loading spinner while fetching data', () => { + renderEntityLink({ + entityManagerMockReturnValue: { + isEntityCentricExperienceViewEnabled: undefined, + isEnablementPending: true, + }, + serviceEntitySummaryMockReturnValue: { + serviceEntitySummary: undefined, + serviceEntitySummaryStatus: FETCH_STATUS.LOADING, + }, + hasApmDataFetcherMockReturnValue: { + data: undefined, + status: FETCH_STATUS.LOADING, + }, + }); + expect(screen.queryByTestId('apmEntityLinkLoadingSpinner')).toBeInTheDocument(); + }); + + it('renders EEM callout when EEM is enabled but service is not found on EEM indices', () => { + renderEntityLink({ + entityManagerMockReturnValue: { + isEntityCentricExperienceViewEnabled: true, + isEnablementPending: false, + }, + serviceEntitySummaryMockReturnValue: { + serviceEntitySummary: undefined, + serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, + }, + hasApmDataFetcherMockReturnValue: { + data: { hasData: false }, + status: FETCH_STATUS.SUCCESS, + }, + }); + + expect(screen.queryByTestId('apmEntityLinkLoadingSpinner')).not.toBeInTheDocument(); + expect(screen.queryByTestId('apmEntityLinkEEMCallout')).toBeInTheDocument(); + }); + + it('renders Service Overview page when EEM is disabled', () => { + renderEntityLink({ + entityManagerMockReturnValue: { + isEntityCentricExperienceViewEnabled: false, + isEnablementPending: false, + }, + serviceEntitySummaryMockReturnValue: { + serviceEntitySummary: undefined, + serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, + }, + hasApmDataFetcherMockReturnValue: { + data: { hasData: false }, + status: FETCH_STATUS.SUCCESS, + }, + query: { + rangeFrom: 'now-1h', + rangeTo: 'now', + }, + }); + + expect(screen.queryByTestId('apmEntityLinkLoadingSpinner')).not.toBeInTheDocument(); + expect(screen.queryByTestId('apmEntityLinkEEMCallout')).not.toBeInTheDocument(); + expect(Redirect).toHaveBeenCalledWith( + expect.objectContaining({ + to: '/services/foo/overview?comparisonEnabled=true&environment=ENVIRONMENT_ALL&kuery=&latencyAggregationType=avg&rangeFrom=now-1h&rangeTo=now&serviceGroup=', + }), + {} + ); + }); + + it('renders Service Overview page when EEM is enabled but Service is not found on EEM but it has raw APM data', () => { + renderEntityLink({ + entityManagerMockReturnValue: { + isEntityCentricExperienceViewEnabled: true, + isEnablementPending: false, + }, + serviceEntitySummaryMockReturnValue: { + serviceEntitySummary: undefined, + serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, + }, + hasApmDataFetcherMockReturnValue: { + data: { hasData: true }, + status: FETCH_STATUS.SUCCESS, + }, + query: { + rangeFrom: 'now-1h', + rangeTo: 'now', + }, + }); + + expect(screen.queryByTestId('apmEntityLinkLoadingSpinner')).not.toBeInTheDocument(); + expect(screen.queryByTestId('apmEntityLinkEEMCallout')).not.toBeInTheDocument(); + expect(Redirect).toHaveBeenCalledWith( + expect.objectContaining({ + to: '/services/foo/overview?comparisonEnabled=true&environment=ENVIRONMENT_ALL&kuery=&latencyAggregationType=avg&rangeFrom=now-1h&rangeTo=now&serviceGroup=', + }), + {} + ); + }); + + it('renders Service Overview page when EEM is enabled and Service is found on EEM', () => { + renderEntityLink({ + entityManagerMockReturnValue: { + isEntityCentricExperienceViewEnabled: true, + isEnablementPending: false, + }, + serviceEntitySummaryMockReturnValue: { + serviceEntitySummary: { dataStreamTypes: ['metrics'] } as unknown as ServiceEntitySummary, + serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, + }, + hasApmDataFetcherMockReturnValue: { + data: { hasData: true }, + status: FETCH_STATUS.SUCCESS, + }, + query: { + rangeFrom: 'now-1h', + rangeTo: 'now', + }, + }); + + expect(screen.queryByTestId('apmEntityLinkLoadingSpinner')).not.toBeInTheDocument(); + expect(screen.queryByTestId('apmEntityLinkEEMCallout')).not.toBeInTheDocument(); + expect(Redirect).toHaveBeenCalledWith( + expect.objectContaining({ + to: '/services/foo/overview?comparisonEnabled=true&environment=ENVIRONMENT_ALL&kuery=&latencyAggregationType=avg&rangeFrom=now-1h&rangeTo=now&serviceGroup=', + }), + {} + ); + }); + + it('renders Service Overview page setting time range from data plugin', () => { + renderEntityLink({ + entityManagerMockReturnValue: { + isEntityCentricExperienceViewEnabled: true, + isEnablementPending: false, + }, + serviceEntitySummaryMockReturnValue: { + serviceEntitySummary: { dataStreamTypes: ['metrics'] } as unknown as ServiceEntitySummary, + serviceEntitySummaryStatus: FETCH_STATUS.SUCCESS, + }, + hasApmDataFetcherMockReturnValue: { + data: { hasData: true }, + status: FETCH_STATUS.SUCCESS, + }, + }); + + expect(screen.queryByTestId('apmEntityLinkLoadingSpinner')).not.toBeInTheDocument(); + expect(screen.queryByTestId('apmEntityLinkEEMCallout')).not.toBeInTheDocument(); + expect(Redirect).toHaveBeenCalledWith( + expect.objectContaining({ + to: '/services/foo/overview?comparisonEnabled=true&environment=ENVIRONMENT_ALL&kuery=&latencyAggregationType=avg&rangeFrom=now-24h&rangeTo=now&serviceGroup=', + }), + {} + ); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/index.tsx new file mode 100644 index 0000000000000..5fdbcc9399258 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/public/components/app/entities/entity_link/index.tsx @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButtonEmpty, EuiEmptyPrompt, EuiImage, EuiLink, EuiLoadingSpinner } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { dashboardsDark, dashboardsLight } from '@kbn/shared-svg'; +import React from 'react'; +import { Redirect } from 'react-router-dom'; +import { ENVIRONMENT_ALL_VALUE } from '../../../../../common/environment_filter_values'; +import { useServiceEntitySummaryFetcher } from '../../../../context/apm_service/use_service_entity_summary_fetcher'; +import { useEntityManagerEnablementContext } from '../../../../context/entity_manager_context/use_entity_manager_enablement_context'; +import { useApmParams } from '../../../../hooks/use_apm_params'; +import { useApmRouter } from '../../../../hooks/use_apm_router'; +import { FETCH_STATUS, isPending, useFetcher } from '../../../../hooks/use_fetcher'; +import { useTheme } from '../../../../hooks/use_theme'; +import { ApmPluginStartDeps } from '../../../../plugin'; + +const pageHeader = { + pageTitle: 'APM', +}; + +export function EntityLink() { + const router = useApmRouter({ prependBasePath: false }); + const theme = useTheme(); + const { services } = useKibana(); + const { observabilityShared, data } = services; + const timeRange = data.query.timefilter.timefilter.getTime(); + const ObservabilityPageTemplate = observabilityShared.navigation.PageTemplate; + const { + path: { serviceName }, + query: { rangeFrom = timeRange.from, rangeTo = timeRange.to }, + } = useApmParams('/link-to/entity/{serviceName}'); + const { isEntityCentricExperienceViewEnabled, isEnablementPending } = + useEntityManagerEnablementContext(); + + const { serviceEntitySummary, serviceEntitySummaryStatus } = useServiceEntitySummaryFetcher({ + serviceName, + environment: ENVIRONMENT_ALL_VALUE, + }); + + const { data: hasApmData, status: hasApmDataStatus } = useFetcher((callApmApi) => { + return callApmApi('GET /internal/apm/has_data'); + }, []); + + if ( + isEnablementPending || + serviceEntitySummaryStatus === FETCH_STATUS.LOADING || + isPending(hasApmDataStatus) + ) { + return ; + } + + if ( + // When EEM is enabled and the service is not found on the EEM indices and there's no APM data, display a callout guiding on the limitations of EEM + isEntityCentricExperienceViewEnabled === true && + (serviceEntitySummary?.dataStreamTypes === undefined || + serviceEntitySummary.dataStreamTypes.length === 0) && + hasApmData?.hasData !== true + ) { + return ( + + + } + title={ +

+ {i18n.translate('xpack.apm.entityLink.eemGuide.title', { + defaultMessage: 'Service not supported', + })} +

+ } + body={ +

+ + {i18n.translate('xpack.apm.entityLink.eemGuide.description.link', { + defaultMessage: 'limitations with the Elastic Entity Model', + })} + + ), + }} + /> +

+ } + actions={[ + { + window.history.back(); + }} + > + {i18n.translate('xpack.apm.entityLink.eemGuide.goBackButtonLabel', { + defaultMessage: 'Go back', + })} + , + ]} + /> + + ); + } + + return ( + + ); +} diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx index 0f370aabe1731..2e91865083b8c 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx @@ -38,19 +38,19 @@ export function ErrorSampleContextualInsight({ instructions: `I'm an SRE. I am looking at an exception and trying to understand what it means. Your task is to describe what the error means and what it could be caused by. - + The error occurred on a service called ${serviceName}, which is a ${runtimeName} service written in ${languageName}. The runtime version is ${runtimeVersion}. - + The request it occurred for is called ${transactionName}. - + ${ logStacktrace ? `The log stacktrace: ${logStacktrace}` : '' } - + ${ exceptionStacktrace ? `The exception stacktrace: diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/index.tsx index 24a30364e266d..74b3975335c90 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/index.tsx @@ -7,13 +7,16 @@ import { EuiEmptyPrompt } from '@elastic/eui'; import React from 'react'; +import { i18n } from '@kbn/i18n'; import { Redirect } from 'react-router-dom'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { getRedirectToTransactionDetailPageUrl } from './get_redirect_to_transaction_detail_page_url'; import { getRedirectToTracePageUrl } from './get_redirect_to_trace_page_url'; import { useApmParams } from '../../../hooks/use_apm_params'; import { useTimeRange } from '../../../hooks/use_time_range'; +import { ApmPluginStartDeps } from '../../../plugin'; const CentralizedContainer = euiStyled.div` height: 100%; @@ -21,9 +24,12 @@ const CentralizedContainer = euiStyled.div` `; export function TraceLink() { + const { services } = useKibana(); + const { data: dataService } = services; + const timeRange = dataService.query.timefilter.timefilter.getTime(); const { path: { traceId }, - query: { rangeFrom, rangeTo }, + query: { rangeFrom = timeRange.from, rangeTo = timeRange.to }, } = useApmParams('/link-to/trace/{traceId}'); const { start, end } = useTimeRange({ @@ -35,15 +41,7 @@ export function TraceLink() { (callApmApi) => { if (traceId) { return callApmApi('GET /internal/apm/traces/{traceId}/root_transaction', { - params: { - path: { - traceId, - }, - query: { - start, - end, - }, - }, + params: { path: { traceId }, query: { start, end } }, }); } }, @@ -62,7 +60,16 @@ export function TraceLink() { return ( - Fetching trace...} /> + + {i18n.translate('xpack.apm.traceLink.fetchingTraceLabel', { + defaultMessage: 'Fetching trace...', + })} + + } + /> ); } diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/trace_link.test.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/trace_link.test.tsx index 432262bb79b11..3250702b0cb80 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/trace_link.test.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/trace_link.test.tsx @@ -18,6 +18,17 @@ import { import * as hooks from '../../../hooks/use_fetcher'; import * as useApmParamsHooks from '../../../hooks/use_apm_params'; +jest.mock('@kbn/kibana-react-plugin/public', () => ({ + ...jest.requireActual('@kbn/kibana-react-plugin/public'), + useKibana: jest.fn().mockReturnValue({ + services: { + data: { + query: { timefilter: { timefilter: { getTime: () => ({ from: 'now-1h', to: 'now' }) } } }, + }, + }, + }), +})); + function Wrapper({ children }: { children?: ReactNode }) { return ( @@ -125,5 +136,20 @@ describe('TraceLink', () => { '/services/foo/transactions/view?traceId=123&transactionId=456&transactionName=bar&transactionType=GET&rangeFrom=now-24h&rangeTo=now&waterfallItemId=' ); }); + + it('sets time range from data plugin when client does not pass it', () => { + jest.spyOn(useApmParamsHooks as any, 'useApmParams').mockReturnValue({ + path: { + traceId: '123', + }, + query: {}, + }); + + const component = shallow(); + + expect(component.prop('to')).toEqual( + '/services/foo/transactions/view?traceId=123&transactionId=456&transactionName=bar&transactionType=GET&rangeFrom=now-1h&rangeTo=now&waterfallItemId=' + ); + }); }); }); diff --git a/x-pack/plugins/observability_solution/apm/public/components/routing/apm_route_config.tsx b/x-pack/plugins/observability_solution/apm/public/components/routing/apm_route_config.tsx index dd82e775d556d..4a83df349035a 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/routing/apm_route_config.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/routing/apm_route_config.tsx @@ -24,6 +24,7 @@ import { ServiceGroupsList } from '../app/service_groups'; import { offsetRt } from '../../../common/comparison_rt'; import { diagnosticsRoute } from '../app/diagnostics'; import { TransactionDetailsByNameLink } from '../app/transaction_details_link'; +import { EntityLink } from '../app/entities/entity_link'; const ServiceGroupsTitle = i18n.translate('xpack.apm.views.serviceGroups.title', { defaultMessage: 'Services', @@ -34,6 +35,18 @@ const ServiceGroupsTitle = i18n.translate('xpack.apm.views.serviceGroups.title', * creates the routes. */ const apmRoutes = { + '/link-to/entity/{serviceName}': { + element: , + params: t.type({ + path: t.type({ + serviceName: t.string, + }), + query: t.partial({ + rangeFrom: t.string, + rangeTo: t.string, + }), + }), + }, '/link-to/transaction': { element: , params: t.type({ diff --git a/x-pack/plugins/observability_solution/apm/public/components/routing/templates/apm_service_template/analyze_data_button.stories.tsx b/x-pack/plugins/observability_solution/apm/public/components/routing/templates/apm_service_template/analyze_data_button.stories.tsx index 4bfb69810a524..3ea8707fe7849 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/routing/templates/apm_service_template/analyze_data_button.stories.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/routing/templates/apm_service_template/analyze_data_button.stories.tsx @@ -8,13 +8,12 @@ import type { Story, DecoratorFn } from '@storybook/react'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; -import { CoreStart } from '@kbn/core/public'; -import { createKibanaReactContext } from '@kbn/kibana-react-plugin/public'; import { ENVIRONMENT_ALL } from '../../../../../common/environment_filter_values'; import { MockApmPluginContextWrapper } from '../../../../context/apm_plugin/mock_apm_plugin_context'; import { APMServiceContext } from '../../../../context/apm_service/apm_service_context'; import { AnalyzeDataButton } from './analyze_data_button'; import { FETCH_STATUS } from '../../../../hooks/use_fetcher'; +import { ApmPluginContextValue } from '../../../../context/apm_plugin/apm_plugin_context'; interface Args { agentName: string; @@ -30,13 +29,6 @@ export default { (StoryComponent, { args }) => { const { agentName, canShowDashboard, environment, serviceName } = args; - const KibanaContext = createKibanaReactContext({ - application: { - capabilities: { dashboard: { show: canShowDashboard } }, - }, - http: { basePath: { get: () => '' } }, - } as unknown as Partial); - return ( - + '' } }, + }, + } as unknown as ApmPluginContextValue + } + > - - - + diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/unified_search_bar/unified_search_bar.test.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/unified_search_bar/unified_search_bar.test.tsx index 4156861f3049d..8a7e71907a62a 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/unified_search_bar/unified_search_bar.test.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/unified_search_bar/unified_search_bar.test.tsx @@ -5,20 +5,19 @@ * 2.0. */ +import { mount } from 'enzyme'; import { createMemoryHistory, MemoryHistory } from 'history'; import React from 'react'; import { useLocation } from 'react-router-dom'; -import { createKibanaReactContext } from '@kbn/kibana-react-plugin/public'; +import { UnifiedSearchBar } from '.'; +import { ApmPluginContextValue } from '../../../context/apm_plugin/apm_plugin_context'; import { MockApmPluginContextWrapper } from '../../../context/apm_plugin/mock_apm_plugin_context'; -import * as useFetcherHook from '../../../hooks/use_fetcher'; +import { UrlParams } from '../../../context/url_params_context/types'; import * as useApmDataViewHook from '../../../hooks/use_adhoc_apm_data_view'; import * as useApmParamsHook from '../../../hooks/use_apm_params'; +import * as useFetcherHook from '../../../hooks/use_fetcher'; import * as useProcessorEventHook from '../../../hooks/use_processor_event'; import { fromQuery } from '../links/url_helpers'; -import { CoreStart } from '@kbn/core/public'; -import { UnifiedSearchBar } from '.'; -import { UrlParams } from '../../../context/url_params_context/types'; -import { mount } from 'enzyme'; jest.mock('react-router-dom', () => ({ ...jest.requireActual('react-router-dom'), @@ -37,41 +36,44 @@ function setup({ urlParams, history }: { urlParams: UrlParams; history: MemoryHi const setTimeSpy = jest.fn(); const setRefreshIntervalSpy = jest.fn(); - const KibanaReactContext = createKibanaReactContext({ - usageCollection: { - reportUiCounter: () => {}, - }, - dataViews: { - get: async () => {}, - }, - data: { - query: { - queryString: { - setQuery: setQuerySpy, - getQuery: getQuerySpy, - clearQuery: clearQuerySpy, - }, - timefilter: { - timefilter: { - setTime: setTimeSpy, - setRefreshInterval: setRefreshIntervalSpy, - }, - }, - }, - }, - } as Partial); - // mock transaction types jest.spyOn(useApmDataViewHook, 'useAdHocApmDataView').mockReturnValue({ dataView: undefined }); jest.spyOn(useFetcherHook, 'useFetcher').mockReturnValue({} as any); const wrapper = mount( - - - - - + {}, + }, + dataViews: { + get: async () => {}, + }, + data: { + query: { + queryString: { + setQuery: setQuerySpy, + getQuery: getQuerySpy, + clearQuery: clearQuerySpy, + }, + timefilter: { + timefilter: { + setTime: setTimeSpy, + setRefreshInterval: setRefreshIntervalSpy, + }, + }, + }, + }, + }, + } as unknown as ApmPluginContextValue + } + > + + ); return { @@ -91,6 +93,11 @@ describe('when kuery is already present in the url, the search bar must reflect jest.spyOn(history, 'push'); jest.spyOn(history, 'replace'); }); + + afterAll(() => { + jest.clearAllMocks(); + }); + jest.spyOn(useProcessorEventHook, 'useProcessorEvent').mockReturnValue(undefined); const search = '?method=json'; diff --git a/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx b/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx index e026dc210df25..961ac7c733e50 100644 --- a/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx +++ b/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx @@ -5,28 +5,32 @@ * 2.0. */ -import React, { ReactNode, useMemo } from 'react'; -import { RouterProvider } from '@kbn/typed-react-router-config'; -import { useHistory } from 'react-router-dom'; -import { createMemoryHistory, History } from 'history'; -import { merge, noop } from 'lodash'; import { coreMock } from '@kbn/core/public/mocks'; -import { UrlService } from '@kbn/share-plugin/common/url_service'; -import { createObservabilityRuleTypeRegistryMock } from '@kbn/observability-plugin/public'; +import { UI_SETTINGS } from '@kbn/data-plugin/common'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { LogsLocatorParams, NodeLogsLocatorParams, TraceLogsLocatorParams, } from '@kbn/logs-shared-plugin/common'; -import { UI_SETTINGS } from '@kbn/data-plugin/common'; import { MlLocatorDefinition } from '@kbn/ml-plugin/public'; -import { enableComparisonByDefault } from '@kbn/observability-plugin/public'; -import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common'; -import { ApmPluginContext, ApmPluginContextValue } from './apm_plugin_context'; +import { + createObservabilityRuleTypeRegistryMock, + enableComparisonByDefault, +} from '@kbn/observability-plugin/public'; +import { UrlService } from '@kbn/share-plugin/common/url_service'; +import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; +import { RouterProvider } from '@kbn/typed-react-router-config'; +import { History, createMemoryHistory } from 'history'; +import { merge, noop } from 'lodash'; +import React, { ReactNode, useMemo } from 'react'; +import { useHistory } from 'react-router-dom'; +import { __IntlProvider as IntlProvider } from '@kbn/i18n-react'; import { ConfigSchema } from '../..'; -import { createCallApmApi } from '../../services/rest/create_call_apm_api'; import { apmRouter } from '../../components/routing/apm_route_config'; +import { createCallApmApi } from '../../services/rest/create_call_apm_api'; +import { ApmPluginContext, ApmPluginContextValue } from './apm_plugin_context'; const coreStart = coreMock.createStart({ basePath: '/basepath' }); @@ -67,6 +71,23 @@ const mockCore = merge({}, coreStart, { return uiSettings[key]; }, }, + data: { + query: { + queryString: { getQuery: jest.fn(), setQuery: jest.fn(), clearQuery: jest.fn() }, + timefilter: { + timefilter: { + setTime: jest.fn(), + setRefreshInterval: jest.fn(), + getTime: jest.fn(), + }, + }, + }, + }, + observabilityShared: { + navigation: { + PageTemplate: ({ children }: { children: React.ReactNode }) =>
{children}
, + }, + }, }); const mockConfig: ConfigSchema = { @@ -203,11 +224,16 @@ export function MockApmPluginContextWrapper({ }) ); }, [history, contextHistory]); + return ( - - - {children} - - + + + + + {children} + + + + ); } diff --git a/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_storybook.tsx b/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_storybook.tsx index a342f84e9c5c3..14d8d4404a719 100644 --- a/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_storybook.tsx +++ b/x-pack/plugins/observability_solution/apm/public/context/apm_plugin/mock_apm_plugin_storybook.tsx @@ -187,7 +187,7 @@ export function MockApmPluginStorybook({ contextMock.core as unknown as Partial ); - const history2 = createMemoryHistory({ + const history = createMemoryHistory({ initialEntries: [routePath || '/services/?rangeFrom=now-15m&rangeTo=now'], }); @@ -197,7 +197,7 @@ export function MockApmPluginStorybook({ - + {children} diff --git a/x-pack/plugins/observability_solution/apm/public/context/entity_manager_context/entity_manager_context.tsx b/x-pack/plugins/observability_solution/apm/public/context/entity_manager_context/entity_manager_context.tsx index c2c3d6c1a57be..93205c907caa0 100644 --- a/x-pack/plugins/observability_solution/apm/public/context/entity_manager_context/entity_manager_context.tsx +++ b/x-pack/plugins/observability_solution/apm/public/context/entity_manager_context/entity_manager_context.tsx @@ -4,21 +4,20 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { createContext } from 'react'; import { entityCentricExperience } from '@kbn/observability-plugin/common'; -import { ENTITY_FETCH_STATUS, useEntityManager } from '../../hooks/use_entity_manager'; -import { useLocalStorage } from '../../hooks/use_local_storage'; -import { useApmPluginContext } from '../apm_plugin/use_apm_plugin_context'; +import React, { createContext } from 'react'; import { SERVICE_INVENTORY_STORAGE_KEY, serviceInventoryViewType$, } from '../../analytics/register_service_inventory_view_type_context'; -import { useKibana } from '../kibana_context/use_kibana'; +import { useLocalStorage } from '../../hooks/use_local_storage'; import { ApmPluginStartDeps, ApmServices } from '../../plugin'; +import { useApmPluginContext } from '../apm_plugin/use_apm_plugin_context'; +import { useKibana } from '../kibana_context/use_kibana'; +import { ENTITY_FETCH_STATUS, useEntityManager } from './use_entity_manager'; export interface EntityManagerEnablementContextValue { isEntityManagerEnabled: boolean; - entityManagerEnablementStatus: ENTITY_FETCH_STATUS; isEnablementPending: boolean; refetch: () => void; serviceInventoryViewLocalStorageSetting: ServiceInventoryView; @@ -55,7 +54,6 @@ export function EntityManagerEnablementContextProvider({ const { services } = useKibana(); const { isEnabled: isEntityManagerEnabled, status, refetch } = useEntityManager(); const [tourState, setTourState] = useLocalStorage('apm.serviceEcoTour', TOUR_INITIAL_STATE); - const [serviceInventoryViewLocalStorageSetting, setServiceInventoryViewLocalStorageSetting] = useLocalStorage(SERVICE_INVENTORY_STORAGE_KEY, ServiceInventoryView.classic); @@ -86,7 +84,6 @@ export function EntityManagerEnablementContextProvider({ ({ ...router, - link: (...args: [any]) => core.http.basePath.prepend('/app/apm' + router.link(...args)), + link: (...args: [any]) => + prependBasePath + ? core.http.basePath.prepend('/app/apm' + router.link(...args)) + : router.link(...args), } as unknown as ApmRouter), - [core.http.basePath, router] + [core.http.basePath, prependBasePath, router] ); } diff --git a/x-pack/plugins/observability_solution/apm/public/plugin.ts b/x-pack/plugins/observability_solution/apm/public/plugin.ts index 9524f328755c0..c429fdaeb9ec5 100644 --- a/x-pack/plugins/observability_solution/apm/public/plugin.ts +++ b/x-pack/plugins/observability_solution/apm/public/plugin.ts @@ -19,7 +19,10 @@ import { PluginInitializerContext, SecurityServiceStart, } from '@kbn/core/public'; -import { EntityManagerPublicPluginSetup } from '@kbn/entityManager-plugin/public'; +import { + EntityManagerPublicPluginSetup, + EntityManagerPublicPluginStart, +} from '@kbn/entityManager-plugin/public'; import type { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; import { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import { DiscoverSetup, DiscoverStart } from '@kbn/discover-plugin/public'; @@ -145,7 +148,7 @@ export interface ApmPluginStartDeps { dashboard: DashboardStart; metricsDataAccess: MetricsDataPluginStart; uiSettings: IUiSettingsClient; - entityManager: EntityManagerPublicPluginSetup; + entityManager: EntityManagerPublicPluginStart; } const applicationsTitle = i18n.translate('xpack.apm.navigation.rootTitle', { diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts index 72fb4c8c7d200..a27bdcc3dc813 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts @@ -164,6 +164,7 @@ export function registerGetApmDatasetInfoFunction({ `, }, }; - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts index 9b80d694151ff..8a95fe9c89869 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts @@ -22,16 +22,16 @@ export function registerGetApmDownstreamDependenciesFunction({ registerFunction( { name: 'get_apm_downstream_dependencies', - description: `Get the downstream dependencies (services or uninstrumented backends) for a - service. This allows you to map the downstream dependency name to a service, by - returning both span.destination.service.resource and service.name. Use this to + description: `Get the downstream dependencies (services or uninstrumented backends) for a + service. This allows you to map the downstream dependency name to a service, by + returning both span.destination.service.resource and service.name. Use this to drilldown further if needed.`, descriptionForUser: i18n.translate( 'xpack.apm.observabilityAiAssistant.functions.registerGetApmDownstreamDependencies.descriptionForUser', { - defaultMessage: `Get the downstream dependencies (services or uninstrumented backends) for a - service. This allows you to map the dowstream dependency name to a service, by - returning both span.destination.service.resource and service.name. Use this to + defaultMessage: `Get the downstream dependencies (services or uninstrumented backends) for a + service. This allows you to map the dowstream dependency name to a service, by + returning both span.destination.service.resource and service.name. Use this to drilldown further if needed.`, } ), @@ -67,6 +67,7 @@ export function registerGetApmDownstreamDependenciesFunction({ randomSampler, }), }; - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts index b24c24425b413..f768c30d8af21 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts @@ -84,6 +84,7 @@ export function registerGetApmServicesListFunction({ arguments: args, }), }; - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts index 63bdbd422c658..dc9152d268adb 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts @@ -138,7 +138,8 @@ export function registerGetApmTimeseriesFunction({ content: timeseries.map((series): Omit => omit(series, 'data')), data: timeseries, }; - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/apm/tsconfig.json b/x-pack/plugins/observability_solution/apm/tsconfig.json index 7de9609dbbafe..9195c2547a71a 100644 --- a/x-pack/plugins/observability_solution/apm/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm/tsconfig.json @@ -130,7 +130,7 @@ "@kbn/entities-schema", "@kbn/serverless", "@kbn/aiops-log-rate-analysis", - "@kbn/router-utils" + "@kbn/router-utils", ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/process_row.tsx b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/process_row.tsx index 17657e9366d7d..93d6b6e8efd91 100644 --- a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/process_row.tsx +++ b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/processes/process_row.tsx @@ -48,7 +48,7 @@ export const ContextualInsightProcessRow = ({ command }: { command: string }) => with the arguments to the process you should then explain its arguments and how they influence the behaviour of the process. If I do not provide any arguments then explain the behaviour of the process when no arguments are provided. - + Here is an example with arguments. Process: metricbeat -c /etc/metricbeat.yml -d autodiscover,kafka -e -system.hostfs=/hostfs Explanation: Metricbeat is part of the Elastic Stack. It is a lightweight shipper that you can install on your diff --git a/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/custom_equation/metric_row_with_agg.tsx b/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/custom_equation/metric_row_with_agg.tsx index 755911503d288..8cdb0c0b43c67 100644 --- a/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/custom_equation/metric_row_with_agg.tsx +++ b/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/custom_equation/metric_row_with_agg.tsx @@ -142,7 +142,7 @@ export function MetricRowWithAgg({ } > )} diff --git a/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/expression_row.tsx b/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/expression_row.tsx index d98b0a69d984f..3bb3641967753 100644 --- a/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/expression_row.tsx +++ b/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/expression_row.tsx @@ -166,7 +166,7 @@ export const ExpressionRow: React.FC = (props) => { ; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index 7f8a2be739dd1..2c65ee6dc06b2 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -157,3 +157,5 @@ export interface ObservabilityAIAssistantScreenContext { actions?: Array>; starterPrompts?: StarterPrompt[]; } + +export type AssistantScope = 'observability' | 'search' | 'all'; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx index acd5e2710c9dd..97dac18d4f733 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx @@ -56,6 +56,7 @@ function ChatContent({ }) { const service = useObservabilityAIAssistant(); const chatService = useObservabilityAIAssistantChatService(); + const { scope } = service; const initialMessagesRef = useRef(initialMessages); @@ -68,6 +69,7 @@ function ChatContent({ initialMessages, persist: false, disableFunctions: true, + scope, }); const lastAssistantResponse = getLastMessageOfType( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts index 1f36b49175eea..7a88e4bd5486b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts @@ -80,6 +80,7 @@ describe('useChat', () => { service: { getScreenContexts: () => [], } as unknown as ObservabilityAIAssistantService, + scope: 'observability', } as UseChatProps, }); }); @@ -109,6 +110,7 @@ describe('useChat', () => { service: { getScreenContexts: () => [], } as unknown as ObservabilityAIAssistantService, + scope: 'observability', } as UseChatProps, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts index 712b102c36f85..b51b33797e285 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts @@ -10,6 +10,7 @@ import { merge } from 'lodash'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { AbortError } from '@kbn/kibana-utils-plugin/common'; import type { NotificationsStart } from '@kbn/core/public'; +import { AssistantScope } from '../../common/types'; import { MessageRole, type Message, @@ -55,6 +56,7 @@ interface UseChatPropsWithoutContext { disableFunctions?: boolean; onConversationUpdate?: (event: ConversationCreateEvent | ConversationUpdateEvent) => void; onChatComplete?: (messages: Message[]) => void; + scope: AssistantScope; } export type UseChatProps = Omit; @@ -70,6 +72,7 @@ function useChatWithoutContext({ onChatComplete, persist, disableFunctions, + scope, }: UseChatPropsWithoutContext): UseChatResult { const [chatState, setChatState] = useState(ChatState.Ready); const systemMessage = useMemo(() => { @@ -161,6 +164,7 @@ function useChatWithoutContext({ disableFunctions: disableFunctions ?? false, signal: abortControllerRef.current.signal, conversationId, + scope, }); function getPendingMessages() { @@ -259,6 +263,7 @@ function useChatWithoutContext({ disableFunctions, service, systemMessage, + scope, ] ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx index b5d85be11dfe2..349f044206267 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx @@ -60,6 +60,7 @@ export const mockService: ObservabilityAIAssistantService = { predefinedConversation$: new Observable(), }, navigate: async () => of(), + scope: 'all', }; function createSetupContract(): ObservabilityAIAssistantPublicSetup { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/plugin.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/plugin.tsx index fd2a60dcdfc3a..05c4bc93cd2ae 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/plugin.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/plugin.tsx @@ -65,6 +65,7 @@ export class ObservabilityAIAssistantPlugin coreStart.application.capabilities.observabilityAIAssistant[ aiAssistantCapabilities.show ] === true, + scope: 'observability', })); const withProviders =

( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.test.ts index 59b3e7c9087d9..9d8338f2d3892 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.test.ts @@ -102,6 +102,7 @@ describe('complete', () => { disableFunctions: false, signal: new AbortController().signal, ...params, + scope: 'all', }, requestCallback ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.ts index 85f5f0397ff31..6e03683b44064 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/complete.ts @@ -43,6 +43,7 @@ export function complete( disableFunctions, signal, instructions, + scope, }: { client: Pick; getScreenContexts: () => ObservabilityAIAssistantScreenContext[]; @@ -65,6 +66,7 @@ export function complete( screenContexts, conversationId, instructions, + scope, }, }, }).pipe(shareReplay()); @@ -131,6 +133,7 @@ export function complete( persist, disableFunctions, instructions, + scope, }, requestCallback ).subscribe(subscriber); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.test.ts index 6d0332602c869..c495d33301882 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.test.ts @@ -70,6 +70,7 @@ describe('createChatService', () => { apiClient: clientSpy, registrations: [], signal: new AbortController().signal, + scope: 'observability', }); }); @@ -79,7 +80,12 @@ describe('createChatService', () => { describe('chat', () => { function chat({ signal }: { signal: AbortSignal } = { signal: new AbortController().signal }) { - return service.chat('my_test', { signal, messages: [], connectorId: '' }); + return service.chat('my_test', { + signal, + messages: [], + connectorId: '', + scope: 'observability', + }); } it('correctly parses a stream of JSON lines', async () => { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts index fc8c06ef55f51..e1829308fe69a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts @@ -23,6 +23,7 @@ import { throwError, timestamp, } from 'rxjs'; +import { AssistantScope } from '../../common/types'; import { ChatCompletionChunkEvent, Message, MessageRole } from '../../common'; import { StreamingChatResponseEventType, @@ -137,19 +138,26 @@ export async function createChatService({ signal: setupAbortSignal, registrations, apiClient, + scope, }: { analytics: AnalyticsServiceStart; signal: AbortSignal; registrations: ChatRegistrationRenderFunction[]; apiClient: ObservabilityAIAssistantAPIClient; + scope: AssistantScope; }): Promise { const functionRegistry: FunctionRegistry = new Map(); const renderFunctionRegistry: Map> = new Map(); const [{ functionDefinitions, systemMessage }] = await Promise.all([ - apiClient('GET /internal/observability_ai_assistant/functions', { + apiClient('GET /internal/observability_ai_assistant/{scope}/functions', { signal: setupAbortSignal, + params: { + path: { + scope, + }, + }, }), ...registrations.map((registration) => { return registration({ @@ -196,6 +204,7 @@ export async function createChatService({ connectorId, functionCall, functions: functions ?? [], + scope, }, }, signal, @@ -228,6 +237,7 @@ export async function createChatService({ signal, client, instructions, + scope, }, ({ params }) => { return callStreamingApi('POST /internal/observability_ai_assistant/chat/complete', { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_service.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_service.ts index 7232078d2efe8..ae95cb26148e9 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_service.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_service.ts @@ -8,7 +8,11 @@ import type { AnalyticsServiceStart, CoreStart } from '@kbn/core/public'; import { compact, without } from 'lodash'; import { BehaviorSubject, debounceTime, filter, lastValueFrom, of, Subject, take } from 'rxjs'; -import type { Message, ObservabilityAIAssistantScreenContext } from '../../common/types'; +import type { + AssistantScope, + Message, + ObservabilityAIAssistantScreenContext, +} from '../../common/types'; import { createFunctionRequestMessage } from '../../common/utils/create_function_request_message'; import { createFunctionResponseMessage } from '../../common/utils/create_function_response_message'; import { createCallObservabilityAIAssistantAPI } from '../api'; @@ -19,10 +23,12 @@ export function createService({ analytics, coreStart, enabled, + scope, }: { analytics: AnalyticsServiceStart; coreStart: CoreStart; enabled: boolean; + scope: AssistantScope; }): ObservabilityAIAssistantService { const apiClient = createCallObservabilityAIAssistantAPI(coreStart); @@ -42,7 +48,7 @@ export function createService({ }, start: async ({ signal }) => { const mod = await import('./create_chat_service'); - return await mod.createChatService({ analytics, apiClient, signal, registrations }); + return await mod.createChatService({ analytics, apiClient, signal, registrations, scope }); }, callApi: apiClient, getScreenContexts() { @@ -89,5 +95,6 @@ export function createService({ }, predefinedConversation$: predefinedConversation$.asObservable(), }, + scope, }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx index d3b52f6803621..f48d7868def87 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx @@ -52,4 +52,5 @@ export const createStorybookService = (): ObservabilityAIAssistantService => ({ predefinedConversation$: new Observable(), }, navigate: async () => of(), + scope: 'observability', }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts index 71a8a7e402748..8b265d433f515 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts @@ -19,6 +19,7 @@ import type { ObservabilityAIAssistantScreenContext, PendingMessage, AdHocInstruction, + AssistantScope, } from '../common/types'; import type { TelemetryEventTypeWithPayload } from './analytics'; import type { ObservabilityAIAssistantAPIClient } from './api'; @@ -52,6 +53,7 @@ export interface ObservabilityAIAssistantChatService { functions?: Array>; functionCall?: string; signal: AbortSignal; + scope: AssistantScope; } ) => Observable; complete: (options: { @@ -67,8 +69,13 @@ export interface ObservabilityAIAssistantChatService { }; signal: AbortSignal; instructions?: AdHocInstruction[]; + scope: AssistantScope; }) => Observable; - getFunctions: (options?: { contexts?: string[]; filter?: string }) => FunctionDefinition[]; + getFunctions: (options?: { + contexts?: string[]; + filter?: string; + scope: AssistantScope; + }) => FunctionDefinition[]; hasFunction: (name: string) => boolean; getSystemMessage: () => Message; hasRenderFunction: (name: string) => boolean; @@ -76,7 +83,8 @@ export interface ObservabilityAIAssistantChatService { name: string, args: string | undefined, response: { data?: string; content?: string }, - onActionClick: ChatActionClickHandler + onActionClick: ChatActionClickHandler, + scope?: AssistantScope ) => React.ReactNode; } @@ -94,6 +102,7 @@ export interface ObservabilityAIAssistantService { getScreenContexts: () => ObservabilityAIAssistantScreenContext[]; conversations: ObservabilityAIAssistantConversationService; navigate: (callback: () => void) => Promise>; + scope: AssistantScope; } export type RenderFunction = (options: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts index fd57968617187..61448d297e4d3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts @@ -115,6 +115,7 @@ export function registerContextFunction({ subscriber.complete(); }); }); - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts index 6008b53dd42c5..71a0cfa4bbde0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts @@ -48,6 +48,7 @@ export function registerElasticsearchFunction({ }); return { content: { response } }; - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts index 0088e35a6f6af..bfe04cb56e8cf 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts @@ -38,6 +38,7 @@ export function registerExecuteConnectorFunction({ ).getActionsClientWithRequest(resources.request); const content = await actionsClient.execute({ actionId: id, params }); return { content }; - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts index 57cac3a4e0c0f..9b20d364ef7d9 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts @@ -94,6 +94,7 @@ export function registerGetDatasetInfoFunction({ stats: relevantFieldNames.stats, }, }; - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts index 5b16b79bd9980..a5333ee1a7ffc 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts @@ -38,7 +38,8 @@ export const registerFunctions: RegistrationCallback = async ({ const isServerless = !!resources.plugins.serverless; - functions.registerInstruction(`You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems. You can help them visualise and analyze data, investigate their systems, perform root cause analysis or identify optimisation opportunities. + functions.registerInstruction({ + instruction: `You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems. You can help them visualise and analyze data, investigate their systems, perform root cause analysis or identify optimisation opportunities. It's very important to not assume what the user is meaning. Ask them for clarification if needed. @@ -48,59 +49,64 @@ export const registerFunctions: RegistrationCallback = async ({ /\". Always put a field value in double quotes. Best: service.name:\"opbeans-go\". Wrong: service.name:opbeans-go. This is very important! You can use Github-flavored Markdown in your responses. If a function returns an array, consider using a Markdown table to format the response. - + Note that ES|QL (the Elasticsearch Query Language which is a new piped language) is the preferred query language. If you want to call a function or tool, only call it a single time per message. Wait until the function has been executed and its results returned to you, before executing the same tool or another tool again if needed. DO NOT UNDER ANY CIRCUMSTANCES USE ES|QL syntax (\`service.name == "foo"\`) with "kqlFilter" (\`service.name:"foo"\`). - + The user is able to change the language which they want you to reply in on the settings page of the AI Assistant for Observability, which can be found in the ${ isServerless ? `Project settings.` : `Stack Management app under the option AI Assistants` }. - If the user asks how to change the language, reply in the same language the user asked in.`); + If the user asks how to change the language, reply in the same language the user asked in.`, + scopes: ['observability'], + }); const { ready: isReady } = await client.getKnowledgeBaseStatus(); - functions.registerInstruction(({ availableFunctionNames }) => { - const instructions: string[] = []; - - if ( - availableFunctionNames.includes(QUERY_FUNCTION_NAME) && - availableFunctionNames.includes(GET_DATASET_INFO_FUNCTION_NAME) - ) { - instructions.push(`You MUST use the "${GET_DATASET_INFO_FUNCTION_NAME}" ${ - functions.hasFunction('get_apm_dataset_info') ? 'or the get_apm_dataset_info' : '' - } function before calling the "${QUERY_FUNCTION_NAME}" or the "changes" functions. - + functions.registerInstruction({ + instruction: ({ availableFunctionNames }) => { + const instructions: string[] = []; + + if ( + availableFunctionNames.includes(QUERY_FUNCTION_NAME) && + availableFunctionNames.includes(GET_DATASET_INFO_FUNCTION_NAME) + ) { + instructions.push(`You MUST use the "${GET_DATASET_INFO_FUNCTION_NAME}" ${ + functions.hasFunction('get_apm_dataset_info') ? 'or the get_apm_dataset_info' : '' + } function before calling the "${QUERY_FUNCTION_NAME}" or the "changes" functions. + If a function requires an index, you MUST use the results from the dataset info functions.`); - } + } - if (availableFunctionNames.includes(GET_DATA_ON_SCREEN_FUNCTION_NAME)) { - instructions.push(`You have access to data on the screen by calling the "${GET_DATA_ON_SCREEN_FUNCTION_NAME}" function. + if (availableFunctionNames.includes(GET_DATA_ON_SCREEN_FUNCTION_NAME)) { + instructions.push(`You have access to data on the screen by calling the "${GET_DATA_ON_SCREEN_FUNCTION_NAME}" function. Use it to help the user understand what they are looking at. A short summary of what they are looking at is available in the return of the "${CONTEXT_FUNCTION_NAME}" function. Data that is compact enough automatically gets included in the response for the "${CONTEXT_FUNCTION_NAME}" function.`); - } + } - if (isReady) { - if (availableFunctionNames.includes(SUMMARIZE_FUNCTION_NAME)) { - instructions.push(`You can use the "${SUMMARIZE_FUNCTION_NAME}" function to store new information you have learned in a knowledge database. + if (isReady) { + if (availableFunctionNames.includes(SUMMARIZE_FUNCTION_NAME)) { + instructions.push(`You can use the "${SUMMARIZE_FUNCTION_NAME}" function to store new information you have learned in a knowledge database. Only use this function when the user asks for it. All summaries MUST be created in English, even if the conversation was carried out in a different language.`); - } - - if (availableFunctionNames.includes(CONTEXT_FUNCTION_NAME)) { + } + + if (availableFunctionNames.includes(CONTEXT_FUNCTION_NAME)) { + instructions.push( + `Additionally, you can use the "${CONTEXT_FUNCTION_NAME}" function to retrieve relevant information from the knowledge database.` + ); + } + } else { instructions.push( - `Additionally, you can use the "${CONTEXT_FUNCTION_NAME}" function to retrieve relevant information from the knowledge database.` + `You do not have a working memory. If the user expects you to remember the previous conversations, tell them they can set up the knowledge base.` ); } - } else { - instructions.push( - `You do not have a working memory. If the user expects you to remember the previous conversations, tell them they can set up the knowledge base.` - ); - } - return instructions.map((instruction) => dedent(instruction)); + return instructions.map((instruction) => dedent(instruction)); + }, + scopes: ['all'], }); if (isReady) { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts index f939e3a79799b..f55a8ba432922 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts @@ -95,6 +95,7 @@ export function registerKibanaFunction({ }).then((response) => { return { content: response.data }; }); - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 8865861d81f45..a4c34c5caa5a3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -86,6 +86,7 @@ export function registerSummarizationFunction({ message: `The document has been stored`, }, })); - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts index d57051cf9fb62..f5f235af12544 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts @@ -10,6 +10,7 @@ import { context as otelContext } from '@opentelemetry/api'; import * as t from 'io-ts'; import { from, map } from 'rxjs'; import { Readable } from 'stream'; +import { AssistantScope } from '../../../common/types'; import { aiAssistantSimulatedFunctionCalling } from '../..'; import { createFunctionResponseMessage } from '../../../common/utils/create_function_response_message'; import { withoutTokenCountEvents } from '../../../common/utils/without_token_count_events'; @@ -20,7 +21,7 @@ import { observableIntoStream } from '../../service/util/observable_into_stream' import { withAssistantSpan } from '../../service/util/with_assistant_span'; import { recallAndScore } from '../../utils/recall/recall_and_score'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; -import { functionRt, messageRt, screenContextRt } from '../runtime_types'; +import { assistantScopeType, functionRt, messageRt, screenContextRt } from '../runtime_types'; import { ObservabilityAIAssistantRouteHandlerResources } from '../types'; const chatCompleteBaseRt = t.type({ @@ -60,6 +61,7 @@ const chatCompleteInternalRt = t.intersection([ t.type({ body: t.type({ screenContexts: t.array(screenContextRt), + scope: assistantScopeType, }), }), ]); @@ -81,10 +83,12 @@ async function initializeChatRequest({ request, plugins: { cloud, actions }, params: { - body: { connectorId }, + body: { connectorId, scope }, }, service, -}: ObservabilityAIAssistantRouteHandlerResources & { params: { body: { connectorId: string } } }) { +}: ObservabilityAIAssistantRouteHandlerResources & { + params: { body: { connectorId: string; scope: AssistantScope } }; +}) { await withAssistantSpan('guard_against_invalid_connector', async () => { const actionsClient = await (await actions.start()).getActionsClientWithRequest(request); @@ -97,7 +101,7 @@ async function initializeChatRequest({ }); const [client, cloudStart, simulateFunctionCalling] = await Promise.all([ - service.getClient({ request }), + service.getClient({ request, scope }), cloud?.start(), (await context.core).uiSettings.client.get(aiAssistantSimulatedFunctionCalling), ]); @@ -132,6 +136,7 @@ const chatRoute = createObservabilityAIAssistantServerRoute({ messages: t.array(messageRt), connectorId: t.string, functions: t.array(functionRt), + scope: assistantScopeType, }), t.partial({ functionCall: t.string, @@ -177,6 +182,7 @@ const chatRecallRoute = createObservabilityAIAssistantServerRoute({ prompt: t.string, context: t.string, connectorId: t.string, + scope: assistantScopeType, }), }), handler: async (resources): Promise => { @@ -304,6 +310,7 @@ const publicChatCompleteRoute = createObservabilityAIAssistantServerRoute({ params: { body: { ...restOfBody, + scope: 'observability', screenContexts: [ { actions, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index fae7077953699..1506db576275d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -12,9 +12,15 @@ import { KnowledgeBaseEntryRole } from '../../../common/types'; import type { RecalledEntry } from '../../service/knowledge_base_service'; import { getSystemMessageFromInstructions } from '../../service/util/get_system_message_from_instructions'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; +import { assistantScopeType } from '../runtime_types'; const getFunctionsRoute = createObservabilityAIAssistantServerRoute({ - endpoint: 'GET /internal/observability_ai_assistant/functions', + endpoint: 'GET /internal/observability_ai_assistant/{scope}/functions', + params: t.type({ + path: t.type({ + scope: assistantScopeType, + }), + }), options: { tags: ['access:ai_assistant'], }, @@ -24,7 +30,13 @@ const getFunctionsRoute = createObservabilityAIAssistantServerRoute({ functionDefinitions: FunctionDefinition[]; systemMessage: string; }> => { - const { service, request } = resources; + const { + service, + request, + params: { + path: { scope }, + }, + } = resources; const controller = new AbortController(); request.events.aborted$.subscribe(() => { @@ -44,14 +56,14 @@ const getFunctionsRoute = createObservabilityAIAssistantServerRoute({ client.getKnowledgeBaseUserInstructions(), ]); - const functionDefinitions = functionClient.getFunctions().map((fn) => fn.definition); + const functionDefinitions = functionClient.getFunctions({ scope }).map((fn) => fn.definition); const availableFunctionNames = functionDefinitions.map((def) => def.name); return { functionDefinitions: functionClient.getFunctions().map((fn) => fn.definition), systemMessage: getSystemMessageFromInstructions({ - applicationInstructions: functionClient.getInstructions(), + applicationInstructions: functionClient.getInstructions(scope), userInstructions, adHocInstructions: [], availableFunctionNames, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/runtime_types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/runtime_types.ts index b7f8b9daa8a58..3ead874f22ca1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/runtime_types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/runtime_types.ts @@ -73,6 +73,12 @@ export const baseConversationRt: t.Type = t.type({ public: toBooleanRt, }); +export const assistantScopeType = t.union([ + t.literal('observability'), + t.literal('search'), + t.literal('all'), +]); + export const conversationCreateRt: t.Type = t.intersection([ baseConversationRt, t.type({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts index 9d6c0dba0b124..ea265c580b50f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts @@ -34,7 +34,8 @@ describe('chatFunctionClient', () => { required: ['foo'], }, }, - respondFn + respondFn, + ['all'] ); }); @@ -49,6 +50,7 @@ describe('chatFunctionClient', () => { messages: [], signal: new AbortController().signal, connectorId: 'foo', + useSimulatedFunctionCalling: false, }); }).rejects.toThrowError(`Function arguments are invalid`); @@ -109,6 +111,7 @@ describe('chatFunctionClient', () => { messages: [], signal: new AbortController().signal, connectorId: 'foo', + useSimulatedFunctionCalling: false, }); expect(result).toEqual({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts index fa1d0e5fd669d..d8fbe456da879 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts @@ -10,13 +10,18 @@ import Ajv, { type ErrorObject, type ValidateFunction } from 'ajv'; import dedent from 'dedent'; import { compact, keyBy } from 'lodash'; import { FunctionVisibility, type FunctionResponse } from '../../../common/functions/types'; -import type { Message, ObservabilityAIAssistantScreenContextRequest } from '../../../common/types'; +import type { + AssistantScope, + Message, + ObservabilityAIAssistantScreenContextRequest, +} from '../../../common/types'; import { filterFunctionDefinitions } from '../../../common/utils/filter_function_definitions'; import type { FunctionCallChatFunction, FunctionHandler, FunctionHandlerRegistry, InstructionOrCallback, + InstructionOrCallbackWithScopes, RegisterFunction, RegisterInstruction, } from '../types'; @@ -34,7 +39,7 @@ const ajv = new Ajv({ export const GET_DATA_ON_SCREEN_FUNCTION_NAME = 'get_data_on_screen'; export class ChatFunctionClient { - private readonly instructions: InstructionOrCallback[] = []; + private readonly instructions: InstructionOrCallbackWithScopes[] = []; private readonly functionRegistry: FunctionHandlerRegistry = new Map(); private readonly validators: Map = new Map(); @@ -73,7 +78,8 @@ export class ChatFunctionClient { return { content: allData.filter((data) => dataNames.includes(data.name)), }; - } + }, + ['all'] ); } @@ -84,11 +90,11 @@ export class ChatFunctionClient { }); } - registerFunction: RegisterFunction = (definition, respond) => { + registerFunction: RegisterFunction = (definition, respond, scopes) => { if (definition.parameters) { this.validators.set(definition.name, ajv.compile(definition.parameters)); } - this.functionRegistry.set(definition.name, { definition, respond }); + this.functionRegistry.set(definition.name, { handler: { definition, respond }, scopes }); }; registerInstruction: RegisterInstruction = (instruction) => { @@ -107,8 +113,12 @@ export class ChatFunctionClient { } } - getInstructions(): InstructionOrCallback[] { - return this.instructions; + getInstructions(scope: AssistantScope): InstructionOrCallback[] { + return this.instructions + .filter( + (instruction) => instruction.scopes.includes(scope) || instruction.scopes.includes('all') + ) + .map((i) => i.instruction); } hasAction(name: string) { @@ -117,10 +127,16 @@ export class ChatFunctionClient { getFunctions({ filter, + scope, }: { filter?: string; + scope?: AssistantScope; } = {}): FunctionHandler[] { - const allFunctions = Array.from(this.functionRegistry.values()); + const allFunctions = Array.from(this.functionRegistry.values()) + .filter(({ handler, scopes }) => + scope ? scopes.includes(scope) || scopes.includes('all') : true + ) + .map(({ handler }) => handler); const functionsByName = keyBy(allFunctions, (definition) => definition.definition.name); @@ -147,6 +163,7 @@ export class ChatFunctionClient { messages, signal, connectorId, + useSimulatedFunctionCalling, }: { chat: FunctionCallChatFunction; name: string; @@ -154,6 +171,7 @@ export class ChatFunctionClient { messages: Message[]; signal: AbortSignal; connectorId: string; + useSimulatedFunctionCalling: boolean; }): Promise { const fn = this.functionRegistry.get(name); @@ -165,13 +183,14 @@ export class ChatFunctionClient { this.validate(name, parsedArguments); - return await fn.respond( + return await fn.handler.respond( { arguments: parsedArguments, messages, screenContexts: this.screenContexts, chat, connectorId, + useSimulatedFunctionCalling, }, signal ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts index a0accea06370b..5a7cf81a40122 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts @@ -187,6 +187,7 @@ describe('Observability AI Assistant client', () => { user: { name: 'johndoe', }, + scope: 'all', }); } @@ -850,6 +851,7 @@ describe('Observability AI Assistant client', () => { }, }, ], + useSimulatedFunctionCalling: false, }); }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index f5839b76effe8..fc14558776434 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -52,6 +52,7 @@ import { type KnowledgeBaseEntry, type Message, type AdHocInstruction, + AssistantScope, } from '../../../common/types'; import { withoutTokenCountEvents } from '../../../common/utils/without_token_count_events'; import { CONTEXT_FUNCTION_NAME } from '../../functions/context'; @@ -100,6 +101,7 @@ export class ObservabilityAIAssistantClient { name: string; }; knowledgeBaseService: KnowledgeBaseService; + scope: AssistantScope; } ) {} @@ -162,7 +164,7 @@ export class ObservabilityAIAssistantClient { complete = ({ functionClient, connectorId, - simulateFunctionCalling, + simulateFunctionCalling = false, instructions: adHocInstructions = [], messages: initialMessages, signal, @@ -215,11 +217,11 @@ export class ObservabilityAIAssistantClient { // this is what we eventually store in the conversation const messagesWithUpdatedSystemMessage = replaceSystemMessage( getSystemMessageFromInstructions({ - applicationInstructions: functionClient.getInstructions(), + applicationInstructions: functionClient.getInstructions(this.dependencies.scope), userInstructions, adHocInstructions, availableFunctionNames: functionClient - .getFunctions() + .getFunctions({ scope: this.dependencies.scope }) .map((fn) => fn.definition.name), }), initialMessages @@ -299,6 +301,8 @@ export class ObservabilityAIAssistantClient { disableFunctions, tracer: completeTracer, connectorId, + scope: this.dependencies.scope, + useSimulatedFunctionCalling: simulateFunctionCalling === true, }) ); }), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts index da172c974e9e2..b91600323d41e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts @@ -28,7 +28,7 @@ import { MessageOrChatEvent, } from '../../../../common/conversation_complete'; import { FunctionVisibility } from '../../../../common/functions/types'; -import { AdHocInstruction, Instruction } from '../../../../common/types'; +import { AdHocInstruction, AssistantScope, Instruction } from '../../../../common/types'; import { createFunctionResponseMessage } from '../../../../common/utils/create_function_response_message'; import { emitWithConcatenatedMessage } from '../../../../common/utils/emit_with_concatenated_message'; import { withoutTokenCountEvents } from '../../../../common/utils/without_token_count_events'; @@ -54,6 +54,7 @@ function executeFunctionAndCatchError({ logger, tracer, connectorId, + useSimulatedFunctionCalling, }: { name: string; args: string | undefined; @@ -64,6 +65,7 @@ function executeFunctionAndCatchError({ logger: Logger; tracer: LangTracer; connectorId: string; + useSimulatedFunctionCalling: boolean; }): Observable { // hide token count events from functions to prevent them from // having to deal with it as well @@ -84,6 +86,7 @@ function executeFunctionAndCatchError({ signal, messages, connectorId, + useSimulatedFunctionCalling, }) ); @@ -181,6 +184,8 @@ export function continueConversation({ disableFunctions, tracer, connectorId, + scope, + useSimulatedFunctionCalling, }: { messages: Message[]; functionClient: ChatFunctionClient; @@ -197,6 +202,8 @@ export function continueConversation({ }; tracer: LangTracer; connectorId: string; + scope: AssistantScope; + useSimulatedFunctionCalling: boolean; }): Observable { let nextFunctionCallsLeft = functionCallsLeft; @@ -210,7 +217,7 @@ export function continueConversation({ const messagesWithUpdatedSystemMessage = replaceSystemMessage( getSystemMessageFromInstructions({ - applicationInstructions: functionClient.getInstructions(), + applicationInstructions: functionClient.getInstructions(scope), userInstructions, adHocInstructions, availableFunctionNames: definitions.map((def) => def.name), @@ -310,6 +317,7 @@ export function continueConversation({ logger, tracer, connectorId, + useSimulatedFunctionCalling, }); } @@ -338,6 +346,8 @@ export function continueConversation({ disableFunctions, tracer, connectorId, + scope, + useSimulatedFunctionCalling, }); }) ) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index c087e5940f0b7..359692809f3a4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -13,6 +13,7 @@ import { getSpaceIdFromPath } from '@kbn/spaces-plugin/common'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; import { once } from 'lodash'; import { + AssistantScope, KnowledgeBaseEntryRole, ObservabilityAIAssistantScreenContextRequest, } from '../../common/types'; @@ -248,8 +249,10 @@ export class ObservabilityAIAssistantService { async getClient({ request, + scope, }: { request: KibanaRequest; + scope?: AssistantScope; }): Promise { const controller = new AbortController(); @@ -288,6 +291,7 @@ export class ObservabilityAIAssistantService { } : undefined, knowledgeBaseService: this.kbService!, + scope: scope || 'all', }); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts index 9ae585af9071c..66510008df967 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts @@ -17,6 +17,7 @@ import type { Message, ObservabilityAIAssistantScreenContextRequest, InstructionOrPlainText, + AssistantScope, } from '../../common/types'; import type { ObservabilityAIAssistantRouteHandlerResources } from '../routes/types'; import { ChatFunctionClient } from './chat_function_client'; @@ -55,6 +56,7 @@ type RespondFunction = ( screenContexts: ObservabilityAIAssistantScreenContextRequest[]; chat: FunctionCallChatFunction; connectorId: string; + useSimulatedFunctionCalling: boolean; }, signal: AbortSignal ) => Promise; @@ -66,13 +68,18 @@ export interface FunctionHandler { export type InstructionOrCallback = InstructionOrPlainText | RegisterInstructionCallback; -type RegisterInstructionCallback = ({ +export interface InstructionOrCallbackWithScopes { + instruction: InstructionOrCallback; + scopes: AssistantScope[]; +} + +export type RegisterInstructionCallback = ({ availableFunctionNames, }: { availableFunctionNames: string[]; }) => InstructionOrPlainText | InstructionOrPlainText[] | undefined; -export type RegisterInstruction = (...instructions: InstructionOrCallback[]) => void; +export type RegisterInstruction = (...instruction: InstructionOrCallbackWithScopes[]) => void; export type RegisterFunction = < TParameters extends CompatibleJSONSchema = any, @@ -80,9 +87,13 @@ export type RegisterFunction = < TArguments = FromSchema >( definition: FunctionDefinition, - respond: RespondFunction + respond: RespondFunction, + scopes: AssistantScope[] ) => void; -export type FunctionHandlerRegistry = Map; +export type FunctionHandlerRegistry = Map< + string, + { handler: FunctionHandler; scopes: AssistantScope[] } +>; export type RegistrationCallback = ({}: { signal: AbortSignal; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/functions/visualize_esql.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/functions/visualize_esql.ts index ebdfbf32abac6..499d885d1ab34 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/functions/visualize_esql.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/common/functions/visualize_esql.ts @@ -47,7 +47,16 @@ export interface VisualizeQueryResponsev1 { }; } -export type VisualizeQueryResponse = VisualizeQueryResponsev0 | VisualizeQueryResponsev1; +export type VisualizeQueryResponsev2 = VisualizeQueryResponsev1 & { + data: { + correctedQuery: string; + }; +}; + +export type VisualizeQueryResponse = + | VisualizeQueryResponsev0 + | VisualizeQueryResponsev1 + | VisualizeQueryResponsev2; export type VisualizeESQLFunctionArguments = FromSchema< (typeof visualizeESQLFunction)['parameters'] diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/functions/visualize_esql.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/functions/visualize_esql.tsx index 404ff9e32a4db..e1889c7bc199a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/functions/visualize_esql.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/functions/visualize_esql.tsx @@ -419,6 +419,11 @@ export function registerVisualizeQueryRenderFunction({ ? typedResponse.content.errorMessages : []; + const correctedQuery = + 'data' in typedResponse && 'correctedQuery' in typedResponse.data + ? typedResponse.data.correctedQuery + : query; + if ('data' in typedResponse && 'userOverrides' in typedResponse.data) { userOverrides = typedResponse.data.userOverrides; } @@ -472,7 +477,7 @@ export function registerVisualizeQueryRenderFunction({ break; } - const trimmedQuery = query.trim(); + const trimmedQuery = correctedQuery.trim(); return ( {}, messages: [], saveTitle: () => {}, + scope: 'all', }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.test.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.test.tsx index 3ebca243a56b3..150847a011207 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.test.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.test.tsx @@ -55,6 +55,7 @@ const mockService: MockedService = { predefinedConversation$: new Observable(), }, navigate: jest.fn().mockReturnValue(of()), + scope: 'all', }; const mockChatService = createMockChatService(); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.ts index 0616e9dbaca32..617b1b302473f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_conversation.ts @@ -63,6 +63,7 @@ export function useConversation({ onConversationUpdate, }: UseConversationProps): UseConversationResult { const service = useObservabilityAIAssistantAppService(); + const { scope } = service; const { services: { @@ -126,6 +127,7 @@ export function useConversation({ onConversationUpdate?.({ conversation: event.conversation }); }, persist: true, + scope, }); const [displayedConversationId, setDisplayedConversationId] = useState(initialConversationId); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/evaluation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/evaluation.ts index 6fa07af24208c..030994fa44acf 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/evaluation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/evaluation.ts @@ -100,6 +100,7 @@ function runEvaluations() { evaluationConnectorId: evaluationConnector.id!, persist: argv.persist, suite: mocha.suite, + scope: 'all', }); const header: string[][] = [ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 61ed156530100..8246a9ceae71f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -18,7 +18,10 @@ import { StreamingChatResponseEvent, StreamingChatResponseEventType, } from '@kbn/observability-ai-assistant-plugin/common'; -import type { ObservabilityAIAssistantScreenContext } from '@kbn/observability-ai-assistant-plugin/common/types'; +import type { + AssistantScope, + ObservabilityAIAssistantScreenContext, +} from '@kbn/observability-ai-assistant-plugin/common/types'; import { throwSerializedChatCompletionErrors } from '@kbn/observability-ai-assistant-plugin/common/utils/throw_serialized_chat_completion_errors'; import { isSupportedConnectorType, @@ -238,11 +241,13 @@ export class KibanaClient { evaluationConnectorId, persist, suite, + scope, }: { connectorId: string; evaluationConnectorId: string; persist: boolean; suite?: Mocha.Suite; + scope: AssistantScope; }): ChatClient { function getMessages(message: string | Array): Array { if (typeof message === 'string') { @@ -370,6 +375,7 @@ export class KibanaClient { connectorId: connectorIdOverride || connectorId, functions: functions.map((fn) => pick(fn, 'name', 'description', 'parameters')), functionCall, + scope, }; return that.axios.post( @@ -459,6 +465,7 @@ export class KibanaClient { connectorId, persist, title: currentTitle, + scope, }, { responseType: 'stream', timeout: NaN } ) @@ -534,7 +541,7 @@ export class KibanaClient { which helps our users make sense of their Observability data. Your goal is to verify whether a conversation between the user and the assistant matches the given criteria. - + For each criterion, calculate a score. Explain your score, by describing what the assistant did right, and describing and quoting what the assistant did wrong, where it could improve, and what the root cause was in case of a failure.`, }, @@ -544,13 +551,13 @@ export class KibanaClient { message: { role: MessageRole.User, content: `Evaluate the conversation according to the following criteria, using the "scores" tool: - + ${criteria.map((criterion, index) => { return `${index}: ${criterion}`; })} - + This is the conversation: - + ${JSON.stringify( messages .filter((msg) => msg.role !== MessageRole.System) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts index 03c61843e702a..1d0056fa2f66c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts @@ -131,13 +131,14 @@ export function registerAlertsFunction({ fields: fields.length === 0 ? defaultFields : fields, }, }; - } + }, + ['observability'] ); functions.registerFunction( { name: 'alerts', - description: `Get alerts for Observability. Make sure get_alerts_dataset_info was called before. + description: `Get alerts for Observability. Make sure get_alerts_dataset_info was called before. Use this to get open (and optionally recovered) alerts for Observability assets, like services, hosts or containers. Display the response in tabular format if appropriate. @@ -220,6 +221,7 @@ export function registerAlertsFunction({ alerts, }, }; - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts index dc0e26ea9c777..71872782e27b0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts @@ -149,6 +149,7 @@ export function registerChangesFunction({ }, }, }; - } + }, + ['observability'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/lens.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/lens.ts index dbae57c08c9e2..bb07d701f1708 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/lens.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/lens.ts @@ -8,9 +8,13 @@ import type { ChatFunctionClient } from '@kbn/observability-ai-assistant-plugin/ import { lensFunctionDefinition } from '../../common/functions/lens'; export function registerLensFunction({ functions }: { functions: ChatFunctionClient }) { - functions.registerFunction(lensFunctionDefinition, async () => { - return { - content: {}, - }; - }); + functions.registerFunction( + lensFunctionDefinition, + async () => { + return { + content: {}, + }; + }, + ['all'] + ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts index 13c2bee278c8a..8f7eb7b6b4e1f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { isChatCompletionChunkEvent, isOutputEvent } from '@kbn/inference-plugin/common'; +import { + correctCommonEsqlMistakes, + isChatCompletionChunkEvent, + isOutputEvent, +} from '@kbn/inference-plugin/common'; import { naturalLanguageToEsql } from '@kbn/inference-plugin/server'; import { FunctionVisibility, @@ -16,6 +20,7 @@ import { import { createFunctionResponseMessage } from '@kbn/observability-ai-assistant-plugin/common/utils/create_function_response_message'; import { map } from 'rxjs'; import { v4 } from 'uuid'; +import { RegisterInstructionCallback } from '@kbn/observability-ai-assistant-plugin/server/service/types'; import type { FunctionRegistrationParameters } from '..'; import { runAndValidateEsqlQuery } from './validate_esql_query'; import { convertMessagesForInference } from '../../../common/convert_messages_for_inference'; @@ -28,7 +33,7 @@ export function registerQueryFunction({ resources, pluginsStart, }: FunctionRegistrationParameters) { - functions.registerInstruction(({ availableFunctionNames }) => + const instruction: RegisterInstructionCallback = ({ availableFunctionNames }) => availableFunctionNames.includes(QUERY_FUNCTION_NAME) ? `You MUST use the "${QUERY_FUNCTION_NAME}" function when the user wants to: - visualize data @@ -47,8 +52,8 @@ export function registerQueryFunction({ When the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt. If the "${EXECUTE_QUERY_NAME}" function has been called, summarize these results for the user. The user does not see a visualization in this case.` - : undefined - ); + : undefined; + functions.registerInstruction({ instruction, scopes: ['all'] }); functions.registerFunction( { @@ -61,7 +66,7 @@ export function registerQueryFunction({ such as a metric or list of things, but does not want to visualize it in a table or chart. You do NOT need to ask permission to execute the query after generating it, use the "${EXECUTE_QUERY_NAME}" function directly instead. - + Do not use when the user just asks for an example.`, parameters: { type: 'object', @@ -74,9 +79,11 @@ export function registerQueryFunction({ } as const, }, async ({ arguments: { query } }) => { + const correctedQuery = correctCommonEsqlMistakes(query).output; + const client = (await resources.context.core).elasticsearch.client.asCurrentUser; const { error, errorMessages, rows, columns } = await runAndValidateEsqlQuery({ - query, + query: correctedQuery, client, }); @@ -96,7 +103,8 @@ export function registerQueryFunction({ rows, }, }; - } + }, + ['all'] ); functions.registerFunction( { @@ -108,7 +116,7 @@ export function registerQueryFunction({ function takes no input.`, visibility: FunctionVisibility.AssistantOnly, }, - async ({ messages, connectorId }, signal) => { + async ({ messages, connectorId, useSimulatedFunctionCalling }, signal) => { const esqlFunctions = functions .getFunctions() .filter( @@ -132,6 +140,7 @@ export function registerQueryFunction({ .concat(esqlFunctions) .map((fn) => [fn.name, { description: fn.description, schema: fn.parameters }]) ), + functionCalling: useSimulatedFunctionCalling ? 'simulated' : 'native', }); const chatMessageId = v4(); @@ -179,6 +188,7 @@ export function registerQueryFunction({ return messageAddEvent; }) ); - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts index ac26846f940e6..1c36d08594521 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts @@ -25,16 +25,20 @@ export async function runAndValidateEsqlQuery({ error?: Error; errorMessages?: string[]; }> { - const { errors } = await validateQuery(query, getAstAndSyntaxErrors, { + const queryWithoutLineBreaks = query.replaceAll(/\n/g, ''); + + const { errors } = await validateQuery(queryWithoutLineBreaks, getAstAndSyntaxErrors, { // setting this to true, we don't want to validate the index / fields existence ignoreOnMissingCallbacks: true, }); - const asCommands = splitIntoCommands(query); + const asCommands = splitIntoCommands(queryWithoutLineBreaks); const errorMessages = errors?.map((error) => { if ('location' in error) { - const commandsUntilEndOfError = splitIntoCommands(query.substring(0, error.location.max)); + const commandsUntilEndOfError = splitIntoCommands( + queryWithoutLineBreaks.substring(0, error.location.max) + ); const lastCompleteCommand = asCommands[commandsUntilEndOfError.length - 1]; if (lastCompleteCommand) { return `Error in ${lastCompleteCommand.command}\n: ${error.text}`; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts index bca5b04e2da06..bda75eafc9ade 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts @@ -5,9 +5,10 @@ * 2.0. */ import { VisualizeESQLUserIntention } from '@kbn/observability-ai-assistant-plugin/common/functions/visualize_esql'; +import { correctCommonEsqlMistakes } from '@kbn/inference-plugin/common'; import { visualizeESQLFunction, - type VisualizeQueryResponsev1, + VisualizeQueryResponsev2, } from '../../common/functions/visualize_esql'; import type { FunctionRegistrationParameters } from '.'; import { runAndValidateEsqlQuery } from './query/validate_esql_query'; @@ -32,12 +33,15 @@ export function registerVisualizeESQLFunction({ }: FunctionRegistrationParameters) { functions.registerFunction( visualizeESQLFunction, - async ({ arguments: { query, intention } }): Promise => { + async ({ arguments: { query, intention } }): Promise => { // errorMessages contains the syntax errors from the client side valdation // error contains the error from the server side validation, it is always one error // and help us identify errors like index not found, field not found etc. + + const correctedQuery = correctCommonEsqlMistakes(query).output; + const { columns, errorMessages, rows, error } = await runAndValidateEsqlQuery({ - query, + query: correctedQuery, client: (await resources.context.core).elasticsearch.client.asCurrentUser, }); @@ -47,6 +51,7 @@ export function registerVisualizeESQLFunction({ data: { columns: columns ?? [], rows: rows ?? [], + correctedQuery, }, content: { message, @@ -56,6 +61,7 @@ export function registerVisualizeESQLFunction({ ], }, }; - } + }, + ['all'] ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts index 34b9dd36ea77f..d99e822484b67 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts @@ -154,7 +154,7 @@ async function executor( } const resources = await initResources(request); - const client = await resources.service.getClient({ request }); + const client = await resources.service.getClient({ request, scope: 'observability' }); const functionClient = await resources.service.getFunctionClient({ signal: new AbortController().signal, resources, @@ -227,7 +227,7 @@ If available, include the link of the conversation at the end of your answer.` role: MessageRole.System, content: getSystemMessageFromInstructions({ availableFunctionNames: functionClient.getFunctions().map((fn) => fn.definition.name), - applicationInstructions: functionClient.getInstructions(), + applicationInstructions: functionClient.getInstructions('observability'), userInstructions: [], adHocInstructions: [], }), diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/pages/firehose.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/pages/firehose.tsx index 638d931997ec3..3f986f080fc3e 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/pages/firehose.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/pages/firehose.tsx @@ -19,7 +19,7 @@ export const FirehosePage = () => ( headlineCopy={i18n.translate( 'xpack.observability_onboarding.experimentalOnboardingFlow.customHeader.firehose.text', { - defaultMessage: 'Setting up Amazon Data Firehose', + defaultMessage: 'Set up Amazon Data Firehose', } )} captionCopy={i18n.translate( @@ -29,6 +29,7 @@ export const FirehosePage = () => ( 'This installation is tailored for setting up Firehose in your Observability project with minimal configuration.', } )} + isTechnicalPreview={true} /> } > diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/auto_refresh_callout.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/auto_refresh_callout.tsx new file mode 100644 index 0000000000000..e4cd98c37ee1c --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/auto_refresh_callout.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiText, + useEuiTheme, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { css } from '@emotion/react'; +import { HAS_DATA_FETCH_INTERVAL } from './utils'; + +export function AutoRefreshCallout() { + const { euiTheme } = useEuiTheme(); + const messageId = useGeneratedHtmlId(); + + return ( + + + + + +

+ {i18n.translate( + 'xpack.observability_onboarding.firehosePanel.autorefreshCalloutLabel', + { + defaultMessage: 'Auto-refreshing every {intervalSeconds} s', + values: { intervalSeconds: Math.round(HAS_DATA_FETCH_INTERVAL / 1000) }, + } + )} +

+ + + + + ); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx index 774f02c23a902..dedc05c701e00 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx @@ -5,7 +5,6 @@ * 2.0. */ -import React from 'react'; import { EuiAccordion, EuiCodeBlock, @@ -14,19 +13,20 @@ import { EuiText, useGeneratedHtmlId, } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import React from 'react'; import { FIREHOSE_CLOUDFORMATION_STACK_NAME, FIREHOSE_LOGS_STREAM_NAME, FIREHOSE_METRICS_STREAM_NAME, } from '../../../../common/aws_firehose'; import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button'; +import { DownloadTemplateCallout } from './download_template_callout'; import { buildCreateStackCommand, buildStackStatusCommand } from './utils'; interface Props { encodedApiKey: string; - onboardingId: string; elasticsearchUrl: string; templateUrl: string; isCopyPrimaryAction: boolean; @@ -57,7 +57,7 @@ export function CreateStackCommandSnippet({

+ +

+ +

@@ -94,7 +98,15 @@ export function CreateStackCommandSnippet({ - + {stackStatusCommand} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx new file mode 100644 index 0000000000000..3c62632b382ad --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButton, EuiSpacer, EuiText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import React from 'react'; +import { + FIREHOSE_CLOUDFORMATION_STACK_NAME, + FIREHOSE_LOGS_STREAM_NAME, + FIREHOSE_METRICS_STREAM_NAME, +} from '../../../../common/aws_firehose'; +import { DownloadTemplateCallout } from './download_template_callout'; +import { buildCreateStackAWSConsoleURL } from './utils'; + +interface Props { + encodedApiKey: string; + elasticsearchUrl: string; + templateUrl: string; + isPrimaryAction: boolean; +} + +export function CreateStackInAWSConsole({ + encodedApiKey, + elasticsearchUrl, + templateUrl, + isPrimaryAction, +}: Props) { + const awsConsoleURL = buildCreateStackAWSConsoleURL({ + templateUrl, + stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME, + logsStreamName: FIREHOSE_LOGS_STREAM_NAME, + metricsStreamName: FIREHOSE_METRICS_STREAM_NAME, + elasticsearchUrl, + encodedApiKey, + }); + + return ( + <> + +

+ +

+

+ +

+ + + + + + {i18n.translate( + 'xpack.observability_onboarding.createStackInAWSConsole.createFirehoseStreamInAWSConsoleButtonLabel', + { defaultMessage: 'Create Firehose Stream in AWS' } + )} + + + ); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/download_template_callout.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/download_template_callout.tsx new file mode 100644 index 0000000000000..9a6e9b97bde84 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/download_template_callout.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { FIREHOSE_CLOUDFORMATION_TEMPLATE_URL } from '../../../../common/aws_firehose'; + +export function DownloadTemplateCallout() { + return ( + + {i18n.translate( + 'xpack.observability_onboarding.firehosePanel.downloadCloudFormationTemplateButtonLabel', + { defaultMessage: 'download and modify the CloudFormation template' } + )} + + ), + }} + /> + ); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx index 520171c835808..00a5b582ba1e7 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx @@ -5,23 +5,57 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useCallback, useState } from 'react'; +import { i18n } from '@kbn/i18n'; import { + EuiButtonGroup, + EuiLink, EuiPanel, EuiSkeletonRectangle, EuiSkeletonText, EuiSpacer, EuiSteps, EuiStepStatus, + EuiText, } from '@elastic/eui'; import useEvent from 'react-use/lib/useEvent'; +import { FormattedMessage } from '@kbn/i18n-react'; import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { EmptyPrompt } from '../shared/empty_prompt'; import { CreateStackCommandSnippet } from './create_stack_command_snippet'; import { VisualizeData } from './visualize_data'; +import { CreateStackInAWSConsole } from './create_stack_in_aws_console'; +import { FeedbackButtons } from '../shared/feedback_buttons'; + +enum CreateStackOption { + AWS_CONSOLE_UI = 'createCloudFormationOptionAWSConsoleUI', + AWS_CLI = 'createCloudFormationOptionAWSCLI', +} + +const OPTIONS = [ + { + id: CreateStackOption.AWS_CONSOLE_UI, + label: i18n.translate( + 'xpack.observability_onboarding.firehosePanel.createStackAWSConsoleOptionLabel', + { + defaultMessage: 'Via AWS Console', + } + ), + }, + { + id: CreateStackOption.AWS_CLI, + label: i18n.translate( + 'xpack.observability_onboarding.firehosePanel.createStackAWSCLIOptionLabel', + { defaultMessage: 'Via AWS CLI' } + ), + }, +]; export function FirehosePanel() { const [windowLostFocus, setWindowLostFocus] = useState(false); + const [selectedOptionId, setSelectedOptionId] = useState( + CreateStackOption.AWS_CONSOLE_UI + ); const { data, status, error, refetch } = useFetcher( (callApi) => { return callApi('POST /internal/observability_onboarding/firehose/flow'); @@ -32,6 +66,10 @@ export function FirehosePanel() { useEvent('blur', () => setWindowLostFocus(true), window); + const onOptionChange = useCallback((id: string) => { + setSelectedOptionId(id as CreateStackOption); + }, []); + if (error !== undefined) { return ; } @@ -41,7 +79,45 @@ export function FirehosePanel() { const steps = [ { - title: 'Create a Firehose delivery stream and ingest CloudWatch logs', + title: i18n.translate('xpack.observability_onboarding.firehosePanel.prerequisitesTitle', { + defaultMessage: 'Prerequisites', + }), + children: ( + <> + +

+ +

+

+ + {i18n.translate( + 'xpack.observability_onboarding.firehosePanel.documentationLinkLabel', + { defaultMessage: 'Check the documentation' } + )} + + ), + }} + /> +

+ + + ), + }, + { + title: 'Create a Firehose delivery stream to ingest CloudWatch logs and metrics', children: ( <> {status !== FETCH_STATUS.SUCCESS && ( @@ -52,13 +128,41 @@ export function FirehosePanel() { )} {status === FETCH_STATUS.SUCCESS && data !== undefined && ( - + <> + + + + + {selectedOptionId === CreateStackOption.AWS_CONSOLE_UI && ( + + )} + + {selectedOptionId === CreateStackOption.AWS_CLI && ( + + )} + )} ), @@ -73,6 +177,7 @@ export function FirehosePanel() { return ( + ); } diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/progress_callout.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/progress_callout.tsx new file mode 100644 index 0000000000000..b520515c2635d --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/progress_callout.tsx @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiText, EuiIconTip, EuiHorizontalRule } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { css } from '@emotion/react'; +import { ProgressIndicator } from '../shared/progress_indicator'; + +const SUPPORTED_SERVICES = [ + 'API Gateway', + 'AWS Usage', + 'CloudTrail', + 'DynamoDB', + 'EBS', + 'EC2', + 'ECS', + 'ELB', + 'EMR', + 'Kinesis Data Stream', + 'Lambda', + 'MSK', + 'NAT Gateway', + 'RDS', + 'Route53', + 'S3', + 'SNS', + 'SQS', + 'VPC', + 'VPN', +]; + +export function ProgressCallout() { + return ( + + +

+ {i18n.translate('xpack.observability_onboarding.firehosePanel.waitingForDataTitle', { + defaultMessage: 'Retrieving data from Amazon Data Firehose', + })} +

+ + + + {i18n.translate( + 'xpack.observability_onboarding.progressCallout.strong.allServicesWeCanLabel', + { defaultMessage: 'All services we can detect' } + )} + + +
    + {SUPPORTED_SERVICES.map((service) => ( +
  • {service}
    • + ))} +
  • + {i18n.translate( + 'xpack.observability_onboarding.progressCallout.li.otherLabel', + { + defaultMessage: + 'Other (Unsupported logs will be stored in a generic Firehose index).', + } + )} +
    • +
+ + } + position="top" + type="iInCircle" + /> + + } + isLoading={true} + css={css` + display: inline-block; + `} + /> + ); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_aws_service_get_started_list.ts b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_aws_service_get_started_list.ts index 2aa08f7a6bed9..277c565986d3c 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_aws_service_get_started_list.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_aws_service_get_started_list.ts @@ -98,7 +98,7 @@ export function useAWSServiceGetStartedList(): AWSServiceGetStartedConfig[] { const generateMetricsDiscoverActionLink = useCallback( (namespace: string, name: string) => ({ - id: `logs-explorer-${namespace}`, + id: `discover-${namespace}`, title: i18n.translate('xpack.observability_onboarding.firehosePanel.exploreDataTitle', { defaultMessage: 'See {name} metrics data in Discover', values: { name }, diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts index 7fdfe5890830b..0fa28276f7fcc 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts @@ -5,6 +5,8 @@ * 2.0. */ +export const HAS_DATA_FETCH_INTERVAL = 5000; + export function buildCreateStackCommand({ templateUrl, stackName, @@ -48,3 +50,41 @@ export function buildStackStatusCommand({ stackName }: { stackName: string }) { .replace(/\n/g, ' ') .replace(/\s\s+/g, ' '); } + +export function buildCreateStackAWSConsoleURL({ + templateUrl, + stackName, + logsStreamName, + metricsStreamName, + elasticsearchUrl, + encodedApiKey, +}: { + templateUrl: string; + stackName: string; + logsStreamName: string; + metricsStreamName: string; + elasticsearchUrl: string; + encodedApiKey: string; +}): string { + const url = new URL('https://console.aws.amazon.com'); + const params = new URLSearchParams({ + templateURL: templateUrl, + stackName, + /** + * 'param_' format is enforced by AWS + * but template parameters are in CamelCase + * which triggers the eslint rule. + */ + /* eslint-disable @typescript-eslint/naming-convention */ + param_FirehoseStreamNameForLogs: logsStreamName, + param_FirehoseStreamNameForMetrics: metricsStreamName, + param_ElasticEndpointURL: elasticsearchUrl, + param_ElasticAPIKey: encodedApiKey, + /* eslint-enable @typescript-eslint/naming-convention */ + }); + + url.pathname = '/cloudformation/home'; + url.hash = `/stacks/quickcreate?${params.toString()}`; + + return url.toString(); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx index aee32dee4fa95..ca16519eb6cc2 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx @@ -5,27 +5,35 @@ * 2.0. */ -import { EuiIcon, EuiSpacer, useEuiTheme, useGeneratedHtmlId } from '@elastic/eui'; -import { css } from '@emotion/react'; +import { EuiIcon, EuiSpacer, EuiText, useGeneratedHtmlId } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import useInterval from 'react-use/lib/useInterval'; +import { union } from 'lodash'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { ObservabilityOnboardingAppServices } from '../../..'; import { FIREHOSE_CLOUDFORMATION_STACK_NAME, FIREHOSE_LOGS_STREAM_NAME, + type AWSIndexName, } from '../../../../common/aws_firehose'; import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { AccordionWithIcon } from '../shared/accordion_with_icon'; import { GetStartedPanel } from '../shared/get_started_panel'; -import { ProgressIndicator } from '../shared/progress_indicator'; import { useAWSServiceGetStartedList } from './use_aws_service_get_started_list'; +import { AutoRefreshCallout } from './auto_refresh_callout'; +import { ProgressCallout } from './progress_callout'; +import { HAS_DATA_FETCH_INTERVAL } from './utils'; -const FETCH_INTERVAL = 2000; const REQUEST_PENDING_STATUS_LIST = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; export function VisualizeData() { const accordionId = useGeneratedHtmlId({ prefix: 'accordion' }); - const { euiTheme } = useEuiTheme(); + const [orderedPopulatedAWSLogsIndexList, setOrderedPopulatedAWSLogsIndexList] = useState< + AWSIndexName[] + >([]); + const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState(true); const { data: populatedAWSLogsIndexList, status, @@ -40,6 +48,49 @@ export function VisualizeData() { }, }); }, []); + const { + services: { notifications }, + } = useKibana(); + + useEffect(() => { + if ( + shouldShowDataReceivedToast && + Array.isArray(populatedAWSLogsIndexList) && + populatedAWSLogsIndexList.length > 0 + ) { + notifications?.toasts.addSuccess( + { + title: i18n.translate( + 'xpack.observability_onboarding.firehosePanel.dataReceivedToastTitle', + { + defaultMessage: 'Your data is on its way', + } + ), + text: i18n.translate( + 'xpack.observability_onboarding.firehosePanel.dataReceivedToastText', + { + defaultMessage: + 'We’ve begun processing your data. In the background, we automatically refresh every few seconds to capture more incoming data.', + } + ), + }, + { + toastLifeTimeMs: 10000, + } + ); + setShouldShowDataReceivedToast(false); + } + + setOrderedPopulatedAWSLogsIndexList((currentList) => + /** + * Using union() to ensure items in the array are unique + * add stay in the insertion order to keep the order of + * the AWS services in the UI. + */ + union(currentList, populatedAWSLogsIndexList) + ); + }, [notifications?.toasts, populatedAWSLogsIndexList, shouldShowDataReceivedToast]); + const awsServiceGetStartedConfigList = useAWSServiceGetStartedList(); useInterval(() => { @@ -48,7 +99,7 @@ export function VisualizeData() { } refetch(); - }, FETCH_INTERVAL); + }, HAS_DATA_FETCH_INTERVAL); if (populatedAWSLogsIndexList === undefined) { return null; @@ -56,58 +107,58 @@ export function VisualizeData() { return ( <> - + +

+ +

+ - + + + {orderedPopulatedAWSLogsIndexList.length === 0 && } + {orderedPopulatedAWSLogsIndexList.length > 0 && } + +
- {awsServiceGetStartedConfigList.map( - ({ id, indexNameList, actionLinks, title, logoURL, previewImage }) => { - const isEnabled = indexNameList.some((indexName) => - populatedAWSLogsIndexList.includes(indexName) - ); - - return ( - } - title={i18n.translate( - 'xpack.observability_onboarding.firehosePanel.awsServiceDataFoundTitle', - { - defaultMessage: '{title}', - values: { title }, - } - )} - extraAction={ - isEnabled ? : null - } - isDisabled={!isEnabled} - css={{ - paddingRight: euiTheme.size.s, - filter: `grayscale(${isEnabled ? 0 : 1})`, - }} - > - - - ); + {orderedPopulatedAWSLogsIndexList.map((indexName, index) => { + const getStartedConfig = awsServiceGetStartedConfigList.find(({ indexNameList }) => + indexNameList.includes(indexName) + ); + + if (!getStartedConfig) { + return null; } - )} + + const { id, actionLinks, title, logoURL, previewImage } = getStartedConfig; + + return ( + } + title={title} + initialIsOpen={true} + borders={ + index === 0 || index === orderedPopulatedAWSLogsIndexList.length - 1 + ? 'none' + : 'horizontal' + } + > + + + ); + })}
); diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx index 2f18c299bbc56..5ac8f2bb05792 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/shared/accordion_with_icon.tsx @@ -15,13 +15,14 @@ import { } from '@elastic/eui'; interface AccordionWithIconProps - extends Omit { + extends Omit { title: string; icon: React.ReactNode; } export const AccordionWithIcon: FunctionComponent = ({ title, icon, + borders = 'horizontal', children, ...rest }) => { @@ -39,7 +40,7 @@ export const AccordionWithIcon: FunctionComponent = ({ } buttonProps={{ paddingSize: 'l' }} - borders="horizontal" + borders={borders} paddingSize="none" >
{children}
diff --git a/x-pack/plugins/observability_solution/observability_shared/common/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/index.ts index ab49080f313ba..82d4bbfe6b3d6 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/index.ts @@ -169,6 +169,10 @@ export type { StacktracesLocator, TopNFunctionsLocatorParams, TopNFunctionsLocator, + ServiceEntityLocator, + ServiceEntityLocatorParams, + TransactionDetailsByTraceIdLocator, + TransactionDetailsByTraceIdLocatorParams, } from './locators'; export { @@ -188,6 +192,10 @@ export { StacktracesLocatorDefinition, TopNFunctionsLocatorDefinition, HOSTS_LOCATOR_ID, + ServiceEntityLocatorDefinition, + SERVICE_ENTITY_LOCATOR, + TransactionDetailsByTraceIdLocatorDefinition, + TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR, } from './locators'; export { COMMON_OBSERVABILITY_GROUPING } from './embeddable_grouping'; diff --git a/x-pack/plugins/observability_solution/observability_shared/common/locators/apm/service_entity_locator.ts b/x-pack/plugins/observability_solution/observability_shared/common/locators/apm/service_entity_locator.ts new file mode 100644 index 0000000000000..3301d0c616231 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_shared/common/locators/apm/service_entity_locator.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { SerializableRecord } from '@kbn/utility-types'; +import type { LocatorDefinition, LocatorPublic } from '@kbn/share-plugin/public'; + +export const SERVICE_ENTITY_LOCATOR = 'SERVICE_ENTITY_LOCATOR'; + +export interface ServiceEntityLocatorParams extends SerializableRecord { + serviceName: string; +} + +export type ServiceEntityLocator = LocatorPublic; + +export class ServiceEntityLocatorDefinition + implements LocatorDefinition +{ + public readonly id = SERVICE_ENTITY_LOCATOR; + + public readonly getLocation = async ({ serviceName }: ServiceEntityLocatorParams) => { + return { + app: 'apm', + path: `/link-to/entity/${encodeURIComponent(serviceName)}`, + state: {}, + }; + }; +} diff --git a/x-pack/plugins/observability_solution/observability_shared/common/locators/apm/transaction_details_by_trace_id_locator.ts b/x-pack/plugins/observability_solution/observability_shared/common/locators/apm/transaction_details_by_trace_id_locator.ts new file mode 100644 index 0000000000000..2e461bc4f9d55 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_shared/common/locators/apm/transaction_details_by_trace_id_locator.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { LocatorDefinition, LocatorPublic } from '@kbn/share-plugin/public'; +import type { SerializableRecord } from '@kbn/utility-types'; + +export const TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR = 'TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR'; + +export interface TransactionDetailsByTraceIdLocatorParams extends SerializableRecord { + traceId: string; +} + +export type TransactionDetailsByTraceIdLocator = + LocatorPublic; + +export class TransactionDetailsByTraceIdLocatorDefinition + implements LocatorDefinition +{ + public readonly id = TRANSACTION_DETAILS_BY_TRACE_ID_LOCATOR; + + public readonly getLocation = async ({ traceId }: TransactionDetailsByTraceIdLocatorParams) => { + return { + app: 'apm', + path: `/link-to/trace/${encodeURIComponent(traceId)}`, + state: {}, + }; + }; +} diff --git a/x-pack/plugins/observability_solution/observability_shared/common/locators/index.ts b/x-pack/plugins/observability_solution/observability_shared/common/locators/index.ts index 98604adc201a2..9c5ded4940d5a 100644 --- a/x-pack/plugins/observability_solution/observability_shared/common/locators/index.ts +++ b/x-pack/plugins/observability_solution/observability_shared/common/locators/index.ts @@ -7,6 +7,8 @@ export * from './apm/service_overview_locator'; export * from './apm/transaction_details_by_name_locator'; +export * from './apm/transaction_details_by_trace_id_locator'; +export * from './apm/service_entity_locator'; export * from './infra/asset_details_flyout_locator'; export * from './infra/asset_details_locator'; export * from './infra/hosts_locator'; diff --git a/x-pack/plugins/observability_solution/observability_shared/public/plugin.ts b/x-pack/plugins/observability_solution/observability_shared/public/plugin.ts index 723ab4f758af4..7cd63d7be7602 100644 --- a/x-pack/plugins/observability_solution/observability_shared/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_shared/public/plugin.ts @@ -31,6 +31,8 @@ import { TopNFunctionsLocatorDefinition, ServiceOverviewLocatorDefinition, TransactionDetailsByNameLocatorDefinition, + ServiceEntityLocatorDefinition, + TransactionDetailsByTraceIdLocatorDefinition, type AssetDetailsFlyoutLocator, type AssetDetailsLocator, type InventoryLocator, @@ -41,6 +43,8 @@ import { type ServiceOverviewLocator, type TransactionDetailsByNameLocator, type MetricsExplorerLocator, + type ServiceEntityLocator, + type TransactionDetailsByTraceIdLocator, } from '../common'; import { updateGlobalNavigation } from './services/update_global_navigation'; export interface ObservabilitySharedSetup { @@ -75,6 +79,8 @@ interface ObservabilitySharedLocators { apm: { serviceOverview: ServiceOverviewLocator; transactionDetailsByName: TransactionDetailsByNameLocator; + transactionDetailsByTraceId: TransactionDetailsByTraceIdLocator; + serviceEntity: ServiceEntityLocator; }; } @@ -148,6 +154,10 @@ export class ObservabilitySharedPlugin implements Plugin { transactionDetailsByName: urlService.locators.create( new TransactionDetailsByNameLocatorDefinition() ), + transactionDetailsByTraceId: urlService.locators.create( + new TransactionDetailsByTraceIdLocatorDefinition() + ), + serviceEntity: urlService.locators.create(new ServiceEntityLocatorDefinition()), }, }; } diff --git a/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/frame_information_ai_assistant.tsx b/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/frame_information_ai_assistant.tsx index fb68611a74b7d..4464a4de4fe7b 100644 --- a/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/frame_information_ai_assistant.tsx +++ b/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/frame_information_ai_assistant.tsx @@ -29,7 +29,7 @@ export function FrameInformationAIAssistant({ frame }: Props) { instructions: `The library is: ${library} The function is: ${functionName} - Your have two tasks. Your first task is to desribe what the library is and what its use cases are, and to + You have two tasks. Your first task is to desribe what the library is and what its use cases are, and to describe what the function does. The output format should look as follows: Library description: Provide a concise description of the library diff --git a/x-pack/plugins/osquery/public/agents/use_osquery_policies.ts b/x-pack/plugins/osquery/public/agents/use_osquery_policies.ts index 52fa6e4da10ea..42edbea3394e0 100644 --- a/x-pack/plugins/osquery/public/agents/use_osquery_policies.ts +++ b/x-pack/plugins/osquery/public/agents/use_osquery_policies.ts @@ -19,12 +19,12 @@ export const useOsqueryPolicies = () => { return useQuery( ['osqueryPolicies'], () => - http.get<{ items: Array<{ policy_id: string }> }>( + http.get<{ items: Array<{ policy_id: string; policy_ids: string[] }> }>( '/internal/osquery/fleet_wrapper/package_policies', { version: API_VERSIONS.internal.v1 } ), { - select: (response) => uniq(response.items.map((p) => p.policy_id)), + select: (response) => uniq(response.items.flatMap((p) => p.policy_ids)), onSuccess: () => setErrorToast(), onError: (error: Error) => setErrorToast(error, { diff --git a/x-pack/plugins/osquery/public/fleet_integration/navigation_buttons.tsx b/x-pack/plugins/osquery/public/fleet_integration/navigation_buttons.tsx index bf6ce0dcc73e8..50d4e9007c634 100644 --- a/x-pack/plugins/osquery/public/fleet_integration/navigation_buttons.tsx +++ b/x-pack/plugins/osquery/public/fleet_integration/navigation_buttons.tsx @@ -14,25 +14,29 @@ import { useKibana, isModifiedEvent, isLeftClickEvent } from '../common/lib/kiba interface NavigationButtonsProps { isDisabled?: boolean; - agentPolicyId?: string | null; + agentPolicyIds?: string[]; } const NavigationButtonsComponent: React.FC = ({ isDisabled = false, - agentPolicyId, + agentPolicyIds, }) => { const { application: { getUrlForApp, navigateToApp }, } = useKibana().services; + const agentPolicyIdsQueryParam = useMemo( + () => agentPolicyIds?.map((id) => `agentPolicyId=${id}`).join('&'), + [agentPolicyIds] + ); const liveQueryHref = useMemo( () => getUrlForApp(PLUGIN_ID, { - path: agentPolicyId - ? `/live_queries/new?agentPolicyId=${agentPolicyId}` + path: agentPolicyIds?.length + ? `/live_queries/new?${agentPolicyIdsQueryParam}` : '/live_queries/new', }), - [agentPolicyId, getUrlForApp] + [agentPolicyIdsQueryParam, agentPolicyIds?.length, getUrlForApp] ); const liveQueryClick = useCallback( @@ -40,13 +44,13 @@ const NavigationButtonsComponent: React.FC = ({ if (!isModifiedEvent(event) && isLeftClickEvent(event)) { event.preventDefault(); navigateToApp(PLUGIN_ID, { - path: agentPolicyId - ? `/live_queries/new?agentPolicyId=${agentPolicyId}` + path: agentPolicyIds?.length + ? `/live_queries/new?${agentPolicyIdsQueryParam}` : '/live_queries/new', }); } }, - [agentPolicyId, navigateToApp] + [agentPolicyIdsQueryParam, agentPolicyIds?.length, navigateToApp] ); const packsHref = getUrlForApp(PLUGIN_ID, { diff --git a/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx b/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx index 732c47fcdcfa7..50dfe808139dc 100644 --- a/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx +++ b/x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx @@ -146,14 +146,21 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo< policy?: PackagePolicyEditExtensionComponentProps['policy']; } >(({ onChange, policy, newPolicy }) => { - const [policyAgentsCount, setPolicyAgentsCount] = useState(null); - const [agentPolicy, setAgentPolicy] = useState(null); + const [agentlessPolicyIds, setAgentlessPolicyIds] = useState([]); + const [agentPolicies, setAgentPolicies] = useState([]); const [editMode] = useState(!!policy); const { application: { getUrlForApp }, http, } = useKibana().services; + const policyIdsWithAgents = useMemo( + () => + agentlessPolicyIds?.length + ? policy?.policy_ids.filter((id) => !agentlessPolicyIds.includes(id)) + : policy?.policy_ids, + [agentlessPolicyIds, policy?.policy_ids] + ); const { form: configForm } = useForm({ defaultValue: { config: JSON.stringify(get(newPolicy, 'inputs[0].config.osquery.value', {}), null, 2), @@ -185,13 +192,16 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo< const [{ config }] = useFormData({ form: configForm, watch: 'config' }); const { isValid, setFieldValue } = configForm; - const agentsLinkHref = useMemo(() => { - if (!policy?.policy_id) return '#'; + const agentsLinkHref = useCallback( + (policyId) => { + if (!policy?.policy_ids?.length) return '#'; - return getUrlForApp(PLUGIN_ID, { - path: pagePathGetters.policy_details({ policyId: policy?.policy_id })[1], - }); - }, [getUrlForApp, policy?.policy_id]); + return getUrlForApp(PLUGIN_ID, { + path: pagePathGetters.policy_details({ policyId })[1], + }); + }, + [getUrlForApp, policy?.policy_ids?.length] + ); const handleConfigUpload = useCallback( (newConfig: any) => { @@ -248,42 +258,57 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo< ); useEffect(() => { - if (editMode && policyAgentsCount === null) { + const policyIdsWithNoAgent: string[] = []; + if (editMode && !agentlessPolicyIds?.length) { const fetchAgentsCount = async () => { try { - const response = await http.fetch<{ results: { total: number } }>( - agentRouteService.getStatusPath(), - { - query: { - policyId: policy?.policy_id, - }, - } - ); - if (response.results) { - setPolicyAgentsCount(response.results.total); + if (policy?.policy_ids?.length) { + await Promise.all( + policy.policy_ids.map(async (id: string) => { + const response = await http.fetch<{ results: { total: number } }>( + agentRouteService.getStatusPath(), + { + query: { + policyId: id, + }, + } + ); + if (response.results.total === 0) { + policyIdsWithNoAgent.push(id); + } + }) + ); + setAgentlessPolicyIds(policyIdsWithNoAgent); } // eslint-disable-next-line no-empty } catch (e) {} }; const fetchAgentPolicyDetails = async () => { - if (policy?.policy_id) { + if (policyIdsWithNoAgent?.length) { + const policiesWithoutAgent: AgentPolicy[] = []; try { - const response = await http.fetch<{ item: AgentPolicy }>( - agentPolicyRouteService.getInfoPath(policy?.policy_id) + await Promise.all( + policyIdsWithNoAgent.map(async (id) => { + const response = await http.fetch<{ item: AgentPolicy }>( + agentPolicyRouteService.getInfoPath(id) + ); + if (response.item) { + policiesWithoutAgent.push(response.item); + } + }) ); - if (response.item) { - setAgentPolicy(response.item); + if (policiesWithoutAgent.length) { + setAgentPolicies(policiesWithoutAgent); } // eslint-disable-next-line no-empty } catch (e) {} } }; - fetchAgentsCount(); - fetchAgentPolicyDetails(); + fetchAgentsCount().then(() => fetchAgentPolicyDetails()); } - }, [editMode, http, policy?.policy_id, policyAgentsCount]); + }, [editMode, http, agentlessPolicyIds?.length, agentlessPolicyIds, policy?.policy_ids]); useEffect(() => { /* @@ -363,21 +388,30 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo< return ( <> {!editMode ? : null} - {policyAgentsCount === 0 ? ( + {agentlessPolicyIds?.length ? ( <>

- {`Fleet has detected that you have not assigned yet any agent to the `} - { - - {agentPolicy?.name ?? policy?.policy_id} - - } + {i18n.translate( + 'xpack.osquery.fleetIntegration.osqueryConfig.noAgentsWarningMessage', + { + defaultMessage: + 'Fleet has detected that you have not assigned yet any agent to the ', + } + )} + {agentPolicies?.map((agentPolicy, index) => ( + + + {agentPolicy.name || agentPolicy?.id} + + {index < agentPolicies.length - 1 && `, `} + + ))} {`. `}
- {`Only agents within the policy with active Osquery Manager integration support the functionality presented below.`} + {`Only agents within the policies with active Osquery Manager integration support the functionality presented below.`}

@@ -385,10 +419,9 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo< ) : null} - {!permissionDenied && ( <> - + { const agentPolicyIds = useMemo(() => { const queryParams = qs.parse(location.search); - return queryParams?.agentPolicyId ? ([queryParams?.agentPolicyId] as string[]) : undefined; + return queryParams?.agentPolicyId + ? isArray(queryParams?.agentPolicyId) + ? queryParams?.agentPolicyId + : [queryParams?.agentPolicyId] + : undefined; }, [location.search]); useEffect(() => { @@ -68,7 +73,7 @@ const NewLiveQueryPageComponent = () => { return ( - + ); }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/legacy_url_conflict_callout.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/legacy_url_conflict_callout.test.tsx index 0582cc13eb92b..5334143f27047 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/legacy_url_conflict_callout.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/legacy_url_conflict_callout.test.tsx @@ -31,6 +31,7 @@ const mockSpacesApi: SpacesApi = { useSpaces: jest.fn(), }, hasOnlyDefaultSpace: false, + isSolutionViewEnabled: true, }; describe('', () => { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/use_redirect_legacy_url.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/use_redirect_legacy_url.test.ts index d7971cce43f80..e706f5fda4b39 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/use_redirect_legacy_url.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/use_redirect_legacy_url.test.ts @@ -32,6 +32,7 @@ const mockSpacesApi: SpacesApi = { useSpaces: jest.fn(), }, hasOnlyDefaultSpace: false, + isSolutionViewEnabled: true, }; describe('useLegacyUrlRedirect', () => { diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts b/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts index d4ae7a0f52b26..83c55cc29178c 100644 --- a/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts @@ -29,6 +29,30 @@ export const buildLargeDocument = ({ return doc; }; +export const buildLargeNestedDocument = ({ + fieldsPerObject, + levels, + fieldSize, +}: { + fieldsPerObject: number; + levels: number; + fieldSize: number; +}): Record => { + if (levels === 1) { + return buildLargeDocument({ numFields: fieldsPerObject, fieldSize }); + } else { + const doc: Record = {}; + range(fieldsPerObject).forEach((idx) => { + doc[`level_${levels}_field${idx}`] = buildLargeNestedDocument({ + fieldsPerObject, + levels: levels - 1, + fieldSize, + }); + }); + return doc; + } +}; + export const addTimestampToDoc = ({ timestamp = new Date(), doc, diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts b/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts index 4448d0cee6897..d061d86fe37dd 100644 --- a/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts @@ -14,7 +14,7 @@ import type { import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; -export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap); +export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap, false); export interface GenerateLargeMappingPropertiesProps { size: number; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 2d3a4cd40b89e..1e5e70a37ae5f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { isEmpty } from 'lodash'; +import { isEmpty, partition } from 'lodash'; import agent from 'elastic-apm-node'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -375,8 +375,17 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ); const legacySignalFields: string[] = Object.keys(aadFieldConversion); + const [ignoreFieldsRegexes, ignoreFieldsStandard] = partition( + [...ignoreFields, ...legacySignalFields], + (field: string) => field.startsWith('/') && field.endsWith('/') + ); + const ignoreFieldsObject: Record = {}; + ignoreFieldsStandard.forEach((field) => { + ignoreFieldsObject[field] = true; + }); const wrapHits = wrapHitsFactory({ - ignoreFields: [...ignoreFields, ...legacySignalFields], + ignoreFields: ignoreFieldsObject, + ignoreFieldsRegexes, mergeStrategy, completeRule, spaceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 2675c3996e865..d865ae6232005 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -12,8 +12,8 @@ import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_pat import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants'; import type { ConfigType } from '../../../../config'; import type { Ancestor, SignalSource, SignalSourceHit } from '../types'; -import { buildAlert, buildAncestors, generateAlertId } from '../factories/utils/build_alert'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { buildAlertFields, buildAncestors, generateAlertId } from '../factories/utils/build_alert'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { EqlSequence } from '../../../../../common/detection_engine/types'; import { generateBuildingBlockIds } from '../factories/utils/generate_building_block_ids'; import type { BuildReasonMessage } from '../utils/reason_formatters'; @@ -59,20 +59,21 @@ export const buildAlertGroupFromSequence = ( let baseAlerts: BaseFieldsLatest[] = []; try { baseAlerts = sequence.events.map((event) => - buildBulkBody( + transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - false, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: false, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - 'placeholder-alert-uuid', // This is overriden below - publicBaseUrl - ) + alertUuid: 'placeholder-alert-uuid', // This is overriden below + publicBaseUrl, + }) ); } catch (error) { ruleExecutionLogger.error(error); @@ -153,16 +154,16 @@ export const buildAlertRoot = ( severity: completeRule.ruleParams.severity, mergedDoc: mergedAlerts as SignalSourceHit, }); - const doc = buildAlert( - wrappedBuildingBlocks, + const doc = buildAlertFields({ + docs: wrappedBuildingBlocks, completeRule, spaceId, reason, indicesToQuery, - 'placeholder-uuid', // These will be overriden below + alertUuid: 'placeholder-uuid', // These will be overriden below publicBaseUrl, // Not necessary now, but when the ID is created ahead of time this can be passed - alertTimestampOverride - ); + alertTimestampOverride, + }); const alertId = generateAlertId(doc); const alertUrl = getAlertDetailsUrl({ alertId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts index b0fa2fd6638fa..762f3cff4ce45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts @@ -16,7 +16,7 @@ import type { ConfigType } from '../../../../config'; import type { CompleteRule, EsqlRuleParams } from '../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { SignalSource } from '../types'; import { generateAlertId } from './utils'; @@ -55,20 +55,21 @@ export const wrapEsqlAlerts = ({ index: i, }); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, - [], + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, + indicesToQuery: [], alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts index 7b180d1adfa62..2a6e9a42acf28 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -19,7 +19,7 @@ import type { ConfigType } from '../../../../config'; import type { CompleteRule, EsqlRuleParams } from '../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { SignalSource } from '../types'; import { getSuppressionAlertFields, getSuppressionTerms } from '../utils'; import { generateAlertId } from './utils'; @@ -73,20 +73,21 @@ export const wrapSuppressedEsqlAlerts = ({ const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, - [], + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, + indicesToQuery: [], alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap new file mode 100644 index 0000000000000..17d8ceb0c50b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap @@ -0,0 +1,223 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`buildAlertFields it creates the expected alert fields 1`] = ` +Object { + "@timestamp": "2020-01-01T00:00:00.000Z", + "event.kind": "signal", + "host.asset.criticality": undefined, + "host.risk.calculated_level": undefined, + "host.risk.calculated_score_norm": undefined, + "kibana.alert.ancestors": Array [ + Object { + "depth": 0, + "id": "d5e8eb51-a6a0-456d-8a15-4b79bfec3d71", + "index": "myFakeSignalIndex", + "rule": undefined, + "type": "event", + }, + ], + "kibana.alert.building_block_type": "default", + "kibana.alert.depth": 1, + "kibana.alert.host.criticality_level": undefined, + "kibana.alert.original_time": "2020-04-20T21:27:45.000Z", + "kibana.alert.reason": "test reason", + "kibana.alert.risk_score": 50, + "kibana.alert.rule.actions": Array [], + "kibana.alert.rule.author": Array [ + "Elastic", + ], + "kibana.alert.rule.building_block_type": "default", + "kibana.alert.rule.consumer": "siem", + "kibana.alert.rule.created_at": "2020-03-27T22:55:59.577Z", + "kibana.alert.rule.created_by": "sample user", + "kibana.alert.rule.description": "Detecting root and admin users", + "kibana.alert.rule.enabled": true, + "kibana.alert.rule.exceptions_list": Array [ + Object { + "id": "some_uuid", + "list_id": "list_id_single", + "namespace_type": "single", + "type": "detection", + }, + Object { + "id": "endpoint_list", + "list_id": "endpoint_list", + "namespace_type": "agnostic", + "type": "endpoint", + }, + ], + "kibana.alert.rule.false_positives": Array [], + "kibana.alert.rule.from": "now-6m", + "kibana.alert.rule.immutable": false, + "kibana.alert.rule.indices": Array [], + "kibana.alert.rule.interval": "5m", + "kibana.alert.rule.license": "Elastic License", + "kibana.alert.rule.max_signals": 10000, + "kibana.alert.rule.meta.someMeta": "someField", + "kibana.alert.rule.name": "rule-name", + "kibana.alert.rule.namespace": undefined, + "kibana.alert.rule.note": "# Investigative notes", + "kibana.alert.rule.parameters": Object { + "alert_suppression": undefined, + "author": Array [ + "Elastic", + ], + "building_block_type": "default", + "data_view_id": undefined, + "description": "Detecting root and admin users", + "exceptions_list": Array [ + Object { + "id": "some_uuid", + "list_id": "list_id_single", + "namespace_type": "single", + "type": "detection", + }, + Object { + "id": "endpoint_list", + "list_id": "endpoint_list", + "namespace_type": "agnostic", + "type": "endpoint", + }, + ], + "false_positives": Array [], + "filters": Array [ + Object { + "query": Object { + "match_phrase": Object { + "host.name": "some-host", + }, + }, + }, + ], + "from": "now-6m", + "immutable": false, + "index": Array [ + "auditbeat-*", + "filebeat-*", + "packetbeat-*", + "winlogbeat-*", + ], + "investigation_fields": undefined, + "language": "kuery", + "license": "Elastic License", + "max_signals": 10000, + "meta": Object { + "someMeta": "someField", + }, + "namespace": undefined, + "note": "# Investigative notes", + "query": "user.name: root or user.name: admin", + "references": Array [ + "http://example.com", + "https://example.com", + ], + "related_integrations": Array [], + "required_fields": Array [], + "response_actions": undefined, + "risk_score": 50, + "risk_score_mapping": Array [], + "rule_id": "rule-1", + "rule_name_override": undefined, + "rule_source": Object { + "type": "internal", + }, + "saved_id": undefined, + "setup": "", + "severity": "high", + "severity_mapping": Array [], + "threat": Array [ + Object { + "framework": "MITRE ATT&CK", + "tactic": Object { + "id": "TA0000", + "name": "test tactic", + "reference": "https://attack.mitre.org/tactics/TA0000/", + }, + "technique": Array [ + Object { + "id": "T0000", + "name": "test technique", + "reference": "https://attack.mitre.org/techniques/T0000/", + "subtechnique": Array [ + Object { + "id": "T0000.000", + "name": "test subtechnique", + "reference": "https://attack.mitre.org/techniques/T0000/000/", + }, + ], + }, + ], + }, + ], + "timeline_id": "some-timeline-id", + "timeline_title": "some-timeline-title", + "timestamp_override": undefined, + "timestamp_override_fallback_disabled": undefined, + "to": "now", + "type": "query", + "version": 1, + }, + "kibana.alert.rule.references": Array [ + "http://example.com", + "https://example.com", + ], + "kibana.alert.rule.risk_score": 50, + "kibana.alert.rule.risk_score_mapping": Array [], + "kibana.alert.rule.rule_id": "rule-1", + "kibana.alert.rule.rule_name_override": undefined, + "kibana.alert.rule.severity": "high", + "kibana.alert.rule.severity_mapping": Array [], + "kibana.alert.rule.tags": Array [ + "some fake tag 1", + "some fake tag 2", + ], + "kibana.alert.rule.threat": Array [ + Object { + "framework": "MITRE ATT&CK", + "tactic": Object { + "id": "TA0000", + "name": "test tactic", + "reference": "https://attack.mitre.org/tactics/TA0000/", + }, + "technique": Array [ + Object { + "id": "T0000", + "name": "test technique", + "reference": "https://attack.mitre.org/techniques/T0000/", + "subtechnique": Array [ + Object { + "id": "T0000.000", + "name": "test subtechnique", + "reference": "https://attack.mitre.org/techniques/T0000/000/", + }, + ], + }, + ], + }, + ], + "kibana.alert.rule.throttle": "no_actions", + "kibana.alert.rule.timeline_id": "some-timeline-id", + "kibana.alert.rule.timeline_title": "some-timeline-title", + "kibana.alert.rule.timestamp_override": undefined, + "kibana.alert.rule.to": "now", + "kibana.alert.rule.type": "query", + "kibana.alert.rule.updated_at": "2020-03-27T22:55:59.577Z", + "kibana.alert.rule.updated_by": "sample user", + "kibana.alert.rule.uuid": "04128c15-0d1b-4716-a4c5-46997ac7f3bd", + "kibana.alert.rule.version": 1, + "kibana.alert.severity": "high", + "kibana.alert.status": "active", + "kibana.alert.url": "test/url/app/security/alerts/redirect/test-uuid?index=.alerts-security.alerts-default×tamp=2020-01-01T00:00:00.000Z", + "kibana.alert.user.criticality_level": undefined, + "kibana.alert.uuid": "test-uuid", + "kibana.alert.workflow_assignee_ids": Array [], + "kibana.alert.workflow_status": "open", + "kibana.alert.workflow_tags": Array [], + "kibana.space_ids": Array [ + "default", + ], + "user.asset.criticality": undefined, + "user.risk.calculated_level": undefined, + "user.risk.calculated_score_norm": undefined, +} +`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts index 4aaa0189eefc4..b7f83106ea0b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -8,42 +8,20 @@ import { ALERT_INSTANCE_ID, ALERT_NAMESPACE, - ALERT_REASON, - ALERT_RISK_SCORE, - ALERT_RULE_CONSUMER, - ALERT_RULE_NAMESPACE, - ALERT_RULE_PARAMETERS, ALERT_RULE_UUID, - ALERT_SEVERITY, - ALERT_STATUS, - ALERT_STATUS_ACTIVE, - ALERT_URL, ALERT_UUID, - ALERT_WORKFLOW_ASSIGNEE_IDS, - ALERT_WORKFLOW_STATUS, - ALERT_WORKFLOW_TAGS, EVENT_ACTION, EVENT_KIND, EVENT_MODULE, - SPACE_IDS, TIMESTAMP, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { sampleDocNoSortIdWithTimestamp } from '../../__mocks__/es_results'; -import { buildAlert, buildParent, buildAncestors, additionalAlertFields } from './build_alert'; +import { buildAlertFields, buildParent, buildAncestors } from './build_alert'; import type { Ancestor, SignalSourceHit } from '../../types'; -import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; -import { DEFAULT_ALERTS_INDEX, SERVER_APP_ID } from '../../../../../../common/constants'; import { EVENT_DATASET } from '../../../../../../common/cti/constants'; -import { - ALERT_ANCESTORS, - ALERT_ORIGINAL_TIME, - ALERT_DEPTH, - ALERT_ORIGINAL_EVENT, - ALERT_BUILDING_BLOCK_TYPE, - ALERT_RULE_INDICES, -} from '../../../../../../common/field_maps/field_names'; +import { ALERT_ANCESTORS, ALERT_DEPTH } from '../../../../../../common/field_maps/field_names'; import { getCompleteRuleMock, getQueryRuleParams } from '../../../rule_schema/mocks'; type SignalDoc = SignalSourceHit & { @@ -51,408 +29,30 @@ type SignalDoc = SignalSourceHit & { _source: Required['_source'] & { [TIMESTAMP]: string }; }; -const SPACE_ID = 'space'; -const reason = 'alert reasonable reason'; -const publicBaseUrl = 'testKibanaBasePath.com'; -const alertUuid = 'test-uuid'; - -describe('buildAlert', () => { +describe('buildAlertFields', () => { beforeEach(() => { jest.clearAllMocks(); }); - test('it builds an alert as expected without original_event if event does not exist', () => { - const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - delete doc._source.event; - const completeRule = getCompleteRuleMock(getQueryRuleParams()); - const alert = { - ...buildAlert( - [doc], - completeRule, - SPACE_ID, - reason, - completeRule.ruleParams.index as string[], - alertUuid, - publicBaseUrl, - undefined - ), - ...additionalAlertFields(doc), - }; - const timestamp = alert[TIMESTAMP]; - const expectedAlertUrl = `${publicBaseUrl}/s/${SPACE_ID}/app/security/alerts/redirect/${alertUuid}?index=${DEFAULT_ALERTS_INDEX}-${SPACE_ID}×tamp=${timestamp}`; - const expected = { - [TIMESTAMP]: timestamp, - [EVENT_KIND]: 'signal', - [SPACE_IDS]: [SPACE_ID], - [ALERT_RULE_CONSUMER]: SERVER_APP_ID, - [ALERT_ANCESTORS]: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', - [ALERT_REASON]: 'alert reasonable reason', - [ALERT_STATUS]: ALERT_STATUS_ACTIVE, - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_BUILDING_BLOCK_TYPE]: 'default', - [ALERT_SEVERITY]: 'high', - [ALERT_RISK_SCORE]: 50, - [ALERT_RULE_PARAMETERS]: { - description: 'Detecting root and admin users', - risk_score: 50, - severity: 'high', - building_block_type: 'default', - note: '# Investigative notes', - license: 'Elastic License', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, - author: ['Elastic'], - false_positives: [], - from: 'now-6m', - rule_id: 'rule-1', - max_signals: 10000, - risk_score_mapping: [], - severity_mapping: [], - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - to: 'now', - references: ['http://example.com', 'https://example.com'], - related_integrations: [], - required_fields: [], - setup: '', - version: 1, - exceptions_list: [ - { - id: 'some_uuid', - list_id: 'list_id_single', - namespace_type: 'single', - type: 'detection', - }, - { - id: 'endpoint_list', - list_id: 'endpoint_list', - namespace_type: 'agnostic', - type: 'endpoint', - }, - ], - immutable: false, - rule_source: { - type: 'internal', - }, - type: 'query', - language: 'kuery', - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - query: 'user.name: root or user.name: admin', - filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], - investigation_fields: undefined, - }, - [ALERT_RULE_INDICES]: completeRule.ruleParams.index, - ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { - actions: [], - author: ['Elastic'], - uuid: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - building_block_type: 'default', - created_at: '2020-03-27T22:55:59.577Z', - updated_at: '2020-03-27T22:55:59.577Z', - created_by: 'sample user', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - immutable: false, - license: 'Elastic License', - meta: { - someMeta: 'someField', - }, - name: 'rule-name', - note: '# Investigative notes', - references: ['http://example.com', 'https://example.com'], - severity: 'high', - severity_mapping: [], - updated_by: 'sample user', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - version: 1, - max_signals: 10000, - risk_score: 50, - risk_score_mapping: [], - rule_id: 'rule-1', - interval: '5m', - exceptions_list: getListArrayMock(), - throttle: 'no_actions', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - }), - [ALERT_DEPTH]: 1, - [ALERT_URL]: expectedAlertUrl, - [ALERT_UUID]: alertUuid, - [ALERT_WORKFLOW_TAGS]: [], - [ALERT_WORKFLOW_ASSIGNEE_IDS]: [], - }; - expect(alert).toEqual(expected); - }); - - test('it builds an alert as expected with original_event if present', () => { + test('it creates the expected alert fields', () => { const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - [EVENT_ACTION]: 'socket_opened', - [EVENT_DATASET]: 'socket', - [EVENT_KIND]: 'event', - [EVENT_MODULE]: 'system', - }, - }; const completeRule = getCompleteRuleMock(getQueryRuleParams()); - const alert = { - ...buildAlert( - [doc], - completeRule, - SPACE_ID, - reason, - completeRule.ruleParams.index as string[], - alertUuid, - publicBaseUrl, - undefined - ), - ...additionalAlertFields(doc), - }; - const timestamp = alert[TIMESTAMP]; - const expectedAlertUrl = `${publicBaseUrl}/s/${SPACE_ID}/app/security/alerts/redirect/${alertUuid}?index=${DEFAULT_ALERTS_INDEX}-${SPACE_ID}×tamp=${timestamp}`; - const expected = { - [TIMESTAMP]: timestamp, - [EVENT_KIND]: 'signal', - [SPACE_IDS]: [SPACE_ID], - [ALERT_RULE_CONSUMER]: SERVER_APP_ID, - [ALERT_ANCESTORS]: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', - [ALERT_RULE_INDICES]: completeRule.ruleParams.index, - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }), - [ALERT_REASON]: 'alert reasonable reason', - [ALERT_STATUS]: ALERT_STATUS_ACTIVE, - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_BUILDING_BLOCK_TYPE]: 'default', - [ALERT_SEVERITY]: 'high', - [ALERT_RISK_SCORE]: 50, - [ALERT_RULE_PARAMETERS]: { - description: 'Detecting root and admin users', - risk_score: 50, - severity: 'high', - building_block_type: 'default', - note: '# Investigative notes', - license: 'Elastic License', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, - author: ['Elastic'], - false_positives: [], - from: 'now-6m', - rule_id: 'rule-1', - max_signals: 10000, - risk_score_mapping: [], - severity_mapping: [], - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - to: 'now', - references: ['http://example.com', 'https://example.com'], - related_integrations: [], - required_fields: [], - setup: '', - version: 1, - exceptions_list: [ - { - id: 'some_uuid', - list_id: 'list_id_single', - namespace_type: 'single', - type: 'detection', - }, - { - id: 'endpoint_list', - list_id: 'endpoint_list', - namespace_type: 'agnostic', - type: 'endpoint', - }, - ], - immutable: false, - rule_source: { - type: 'internal', - }, - type: 'query', - language: 'kuery', - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - query: 'user.name: root or user.name: admin', - filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], - investigation_fields: undefined, - }, - ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { - actions: [], - author: ['Elastic'], - uuid: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - building_block_type: 'default', - created_at: '2020-03-27T22:55:59.577Z', - updated_at: '2020-03-27T22:55:59.577Z', - created_by: 'sample user', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - immutable: false, - license: 'Elastic License', - meta: { - someMeta: 'someField', - }, - name: 'rule-name', - note: '# Investigative notes', - references: ['http://example.com', 'https://example.com'], - severity: 'high', - severity_mapping: [], - updated_by: 'sample user', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - version: 1, - max_signals: 10000, - risk_score: 50, - risk_score_mapping: [], - rule_id: 'rule-1', - interval: '5m', - exceptions_list: getListArrayMock(), - throttle: 'no_actions', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - }), - [ALERT_DEPTH]: 1, - [ALERT_URL]: expectedAlertUrl, - [ALERT_UUID]: alertUuid, - [ALERT_WORKFLOW_TAGS]: [], - [ALERT_WORKFLOW_ASSIGNEE_IDS]: [], - }; - expect(alert).toEqual(expected); + const alertFields = buildAlertFields({ + docs: [sampleDoc], + completeRule, + spaceId: 'default', + reason: 'test reason', + indicesToQuery: [], + alertUuid: 'test-uuid', + publicBaseUrl: 'test/url', + alertTimestampOverride: new Date('2020-01-01T00:00:00.000Z'), + }); + expect(alertFields).toMatchSnapshot(); }); test('it builds a parent correctly if the parent does not exist', () => { const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - [EVENT_ACTION]: 'socket_opened', - [EVENT_DATASET]: 'socket', - [EVENT_KIND]: 'event', - [EVENT_MODULE]: 'system', - }, - }; - const parent = buildParent(doc); + const parent = buildParent(sampleDoc); const expected: Ancestor = { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index d81fe7d020282..036ba8c9a644a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -49,7 +49,7 @@ import { requiredOptional } from '@kbn/zod-helpers'; import { createHash } from 'crypto'; import { getAlertDetailsUrl } from '../../../../../../common/utils/alert_detail_path'; -import type { BaseSignalHit, SimpleHit } from '../../types'; +import type { SimpleHit } from '../../types'; import type { ThresholdResult } from '../../threshold/types'; import { getField, @@ -63,8 +63,6 @@ import { ALERT_ANCESTORS, ALERT_DEPTH, ALERT_ORIGINAL_TIME, - ALERT_THRESHOLD_RESULT, - ALERT_ORIGINAL_EVENT, ALERT_BUILDING_BLOCK_TYPE, ALERT_RULE_ACTIONS, ALERT_RULE_INDICES, @@ -97,6 +95,22 @@ import type { BaseFieldsLatest, } from '../../../../../../common/api/detection_engine/model/alerts'; +export interface BuildAlertFieldsProps { + docs: SimpleHit[]; + completeRule: CompleteRule; + spaceId: string | null | undefined; + reason: string; + indicesToQuery: string[]; + alertUuid: string; + publicBaseUrl: string | undefined; + alertTimestampOverride: Date | undefined; + overrides?: { + nameOverride: string; + severityOverride: string; + riskScoreOverride: number; + }; +} + export const generateAlertId = (alert: BaseFieldsLatest) => { return createHash('sha256') .update( @@ -145,21 +159,17 @@ export const buildAncestors = (doc: SimpleHit): AncestorLatest[] => { * @param reason Human readable string summarizing alert. * @param indicesToQuery Array of index patterns searched by the rule. */ -export const buildAlert = ( - docs: SimpleHit[], - completeRule: CompleteRule, - spaceId: string | null | undefined, - reason: string, - indicesToQuery: string[], - alertUuid: string, - publicBaseUrl: string | undefined, - alertTimestampOverride: Date | undefined, - overrides?: { - nameOverride: string; - severityOverride: string; - riskScoreOverride: number; - } -): BaseFieldsLatest => { +export const buildAlertFields = ({ + docs, + completeRule, + spaceId, + reason, + indicesToQuery, + alertUuid, + publicBaseUrl, + alertTimestampOverride, + overrides, +}: BuildAlertFieldsProps): BaseFieldsLatest => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce( @@ -276,28 +286,8 @@ export const buildAlert = ( }; }; -const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => { +export const isThresholdResult = ( + thresholdResult: SearchTypes +): thresholdResult is ThresholdResult => { return typeof thresholdResult === 'object'; }; - -/** - * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. - * We copy the original time from the document as "original_time" since we override the timestamp with the current date time. - * @param doc The parent signal/event of the new signal to be built. - */ -export const additionalAlertFields = (doc: BaseSignalHit) => { - const thresholdResult = doc._source?.threshold_result; - if (thresholdResult != null && !isThresholdResult(thresholdResult)) { - throw new Error(`threshold_result failed to validate: ${thresholdResult}`); - } - const additionalFields: Record = { - ...(thresholdResult != null ? { [ALERT_THRESHOLD_RESULT]: thresholdResult } : {}), - }; - - for (const [key, val] of Object.entries(doc._source ?? {})) { - if (key.startsWith('event.')) { - additionalFields[`${ALERT_ORIGINAL_EVENT}.${key.replace('event.', '')}`] = val; - } - } - return additionalFields; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts deleted file mode 100644 index b2426ceda9767..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { sampleDocWithNonEcsCompliantFields } from '../../__mocks__/es_results'; -import { buildBulkBody } from './build_bulk_body'; -import { getCompleteRuleMock, getEsqlRuleParams } from '../../../rule_schema/mocks'; -import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; - -const SPACE_ID = 'space'; -const publicBaseUrl = 'testKibanaBasePath.com'; -const alertUuid = 'test-uuid'; -const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; -const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); - -describe('buildBulkBody', () => { - test('should strip non-ECS compliant sub-fields of `event.action` field', () => { - const doc = sampleDocWithNonEcsCompliantFields(docId, { - 'event.action': 'process', - 'event.action.keyword': 'process', - }); - const completeRule = getCompleteRuleMock(getEsqlRuleParams()); - const buildReasonMessageStub = jest.fn(); - const alert = buildBulkBody( - SPACE_ID, - completeRule, - doc, - 'missingFields', - [], - true, - buildReasonMessageStub, - [], - undefined, - ruleExecutionLogger, - alertUuid, - publicBaseUrl - ); - - expect(alert['kibana.alert.original_event.action']).toEqual('process'); - expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined(); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts deleted file mode 100644 index 9294cc7159c12..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { flattenWithPrefix } from '@kbn/securitysolution-rules'; -import type * as estypes from '@elastic/elasticsearch/lib/api/types'; -import { requiredOptional } from '@kbn/zod-helpers'; - -import type { BaseHit, SearchTypes } from '../../../../../../common/detection_engine/types'; -import type { ConfigType } from '../../../../../config'; -import type { BuildReasonMessage } from '../../utils/reason_formatters'; -import { getMergeStrategy } from '../../utils/source_fields_merging/strategies'; -import type { BaseSignalHit, SignalSource, SignalSourceHit } from '../../types'; -import { additionalAlertFields, buildAlert } from './build_alert'; -import { filterSource } from './filter_source'; -import type { CompleteRule, RuleParams } from '../../../rule_schema'; -import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; -import { buildRuleNameFromMapping } from '../../utils/mappings/build_rule_name_from_mapping'; -import { buildSeverityFromMapping } from '../../utils/mappings/build_severity_from_mapping'; -import { buildRiskScoreFromMapping } from '../../utils/mappings/build_risk_score_from_mapping'; -import type { BaseFieldsLatest } from '../../../../../../common/api/detection_engine/model/alerts'; -import { stripNonEcsFields } from './strip_non_ecs_fields'; - -const isSourceDoc = ( - hit: SignalSourceHit -): hit is BaseHit<{ '@timestamp': string; _source: SignalSource }> => { - return hit._source != null; -}; - -const buildEventTypeAlert = (doc: BaseSignalHit): Record => { - if (doc._source?.event != null && doc._source?.event instanceof Object) { - return flattenWithPrefix('event', doc._source?.event ?? {}); - } - return {}; -}; - -/** - * Formats the search_after result for insertion into the signals index. We first create a - * "best effort" merged "fields" with the "_source" object, then build the signal object, - * then the event object, and finally we strip away any additional temporary data that was added - * such as the "threshold_result". - * @param completeRule The rule saved object to build overrides - * @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result" - * @returns The body that can be added to a bulk call for inserting the signal. - */ -export const buildBulkBody = ( - spaceId: string | null | undefined, - completeRule: CompleteRule, - doc: estypes.SearchHit, - mergeStrategy: ConfigType['alertMergeStrategy'], - ignoreFields: ConfigType['alertIgnoreFields'], - applyOverrides: boolean, - buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[], - alertTimestampOverride: Date | undefined, - ruleExecutionLogger: IRuleExecutionLogForExecutors, - alertUuid: string, - publicBaseUrl?: string -): BaseFieldsLatest => { - const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); - - const eventFields = buildEventTypeAlert(mergedDoc); - const { result: validatedEventFields, removed: removedEventFields } = - stripNonEcsFields(eventFields); - - const filteredSource = filterSource(mergedDoc); - const { result: validatedSource, removed: removedSourceFields } = - stripNonEcsFields(filteredSource); - - if (removedEventFields.length || removedSourceFields.length) { - ruleExecutionLogger?.debug( - 'Following fields were removed from alert source as ECS non-compliant:', - JSON.stringify(removedSourceFields), - JSON.stringify(removedEventFields) - ); - } - - const overrides = applyOverrides - ? { - nameOverride: buildRuleNameFromMapping({ - eventSource: mergedDoc._source ?? {}, - ruleName: completeRule.ruleConfig.name, - ruleNameMapping: completeRule.ruleParams.ruleNameOverride, - }).ruleName, - severityOverride: buildSeverityFromMapping({ - eventSource: mergedDoc._source ?? {}, - severity: completeRule.ruleParams.severity, - severityMapping: completeRule.ruleParams.severityMapping, - }).severity, - riskScoreOverride: buildRiskScoreFromMapping({ - eventSource: mergedDoc._source ?? {}, - riskScore: completeRule.ruleParams.riskScore, - riskScoreMapping: requiredOptional(completeRule.ruleParams.riskScoreMapping), - }).riskScore, - } - : undefined; - - const reason = buildReasonMessage({ - name: overrides?.nameOverride ?? completeRule.ruleConfig.name, - severity: overrides?.severityOverride ?? completeRule.ruleParams.severity, - mergedDoc, - }); - - const thresholdResult = mergedDoc._source?.threshold_result; - if (isSourceDoc(mergedDoc)) { - return { - ...validatedSource, - ...validatedEventFields, - ...buildAlert( - [mergedDoc], - completeRule, - spaceId, - reason, - indicesToQuery, - alertUuid, - publicBaseUrl, - alertTimestampOverride, - overrides - ), - ...additionalAlertFields({ - ...mergedDoc, - _source: { - ...validatedSource, - ...validatedEventFields, - threshold_result: thresholdResult, - }, - }), - }; - } - - throw Error('Error building alert from source document.'); -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.test.ts deleted file mode 100644 index 73717a7c0a5ec..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { filterSource } from './filter_source'; - -describe('filterSource', () => { - test('should remove keys starting with kibana without modifying the original doc', () => { - const testDoc = { - _index: '', - _id: '', - _source: { - 'kibana.alert.suppression.docs_count': 5, - 'host.name': 'test-host', - }, - }; - const filtered = filterSource(testDoc); - expect(filtered).toEqual({ - 'host.name': 'test-host', - }); - expect(testDoc).toEqual({ - _index: '', - _id: '', - _source: { - 'kibana.alert.suppression.docs_count': 5, - 'host.name': 'test-host', - }, - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts deleted file mode 100644 index c0172207c7a60..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ALERT_THRESHOLD_RESULT } from '../../../../../../common/field_maps/field_names'; -import type { SignalSourceHit } from '../../types'; - -export const filterSource = (doc: SignalSourceHit) => { - const docSource = doc._source ?? {}; - const { - event, - kibana, - signal, - threshold_result: siemSignalsThresholdResult, - [ALERT_THRESHOLD_RESULT]: alertThresholdResult, - ...filteredSource - } = docSource || { - event: null, - kibana: null, - signal: null, - threshold_result: null, - [ALERT_THRESHOLD_RESULT]: null, - }; - - Object.keys(filteredSource).forEach((key) => { - if (key.startsWith('kibana')) { - delete filteredSource[key]; - } - }); - - return filteredSource; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts new file mode 100644 index 0000000000000..92d02e1b3ac4a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts @@ -0,0 +1,438 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + ALERT_REASON, + ALERT_RISK_SCORE, + ALERT_RULE_CONSUMER, + ALERT_RULE_NAMESPACE, + ALERT_RULE_PARAMETERS, + ALERT_SEVERITY, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_URL, + ALERT_UUID, + ALERT_WORKFLOW_ASSIGNEE_IDS, + ALERT_WORKFLOW_STATUS, + ALERT_WORKFLOW_TAGS, + EVENT_ACTION, + EVENT_KIND, + EVENT_MODULE, + SPACE_IDS, + TIMESTAMP, +} from '@kbn/rule-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; + +import { + sampleDocNoSortIdWithTimestamp, + sampleDocWithNonEcsCompliantFields, +} from '../../__mocks__/es_results'; +import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; +import { DEFAULT_ALERTS_INDEX, SERVER_APP_ID } from '../../../../../../common/constants'; +import { EVENT_DATASET } from '../../../../../../common/cti/constants'; +import { + ALERT_ANCESTORS, + ALERT_ORIGINAL_TIME, + ALERT_DEPTH, + ALERT_ORIGINAL_EVENT, + ALERT_BUILDING_BLOCK_TYPE, + ALERT_RULE_INDICES, +} from '../../../../../../common/field_maps/field_names'; + +import { transformHitToAlert } from './transform_hit_to_alert'; +import { + getCompleteRuleMock, + getEsqlRuleParams, + getQueryRuleParams, +} from '../../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; +import { get } from 'lodash'; + +const SPACE_ID = 'space'; +const publicBaseUrl = 'testKibanaBasePath.com'; +const alertUuid = 'test-uuid'; +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); +const buildReasonMessageStub = jest.fn(); + +describe('transformHitToAlert', () => { + it('should strip non-ECS compliant sub-fields of `event.action` field', () => { + const doc = sampleDocWithNonEcsCompliantFields(docId, { + 'event.action': 'process', + 'event.action.keyword': 'process', + }); + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(alert['kibana.alert.original_event.action']).toEqual('process'); + expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined(); + }); + + it('should unset an existing event.kind field in nested notation', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + event: { + kind: 'test-value', + }, + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(get(alert.event, 'kind')).toEqual(undefined); + expect(alert['event.kind']).toEqual('signal'); + }); + + it('should replace an existing event.kind in dot notation', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + 'event.kind': 'test-value', + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(get(alert.event, 'kind')).toEqual(undefined); + expect(alert['event.kind']).toEqual('signal'); + }); + + it('should not add an empty event object if event.kind does not exist', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + testField: 'testValue', + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(alert.event).toEqual(undefined); + expect(alert['event.kind']).toEqual('signal'); + }); + + it('should only copy ECS compatible array elements from event subfields to kibana.alert.original_event', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + 'event.action': ['process', { objectSubfield: 'test' }], + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(alert['kibana.alert.original_event.action']).toEqual(['process']); + }); + + it('builds an alert as expected without original_event if event does not exist', () => { + const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const completeRule = getCompleteRuleMock(getQueryRuleParams()); + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + delete doc._source.event; + + const timestamp = alert[TIMESTAMP]; + const expectedAlertUrl = `${publicBaseUrl}/s/${SPACE_ID}/app/security/alerts/redirect/${alertUuid}?index=${DEFAULT_ALERTS_INDEX}-${SPACE_ID}×tamp=${timestamp}`; + const expected = { + [TIMESTAMP]: timestamp, + [EVENT_KIND]: 'signal', + [SPACE_IDS]: [SPACE_ID], + [ALERT_RULE_CONSUMER]: SERVER_APP_ID, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', + [ALERT_REASON]: undefined, + [ALERT_STATUS]: ALERT_STATUS_ACTIVE, + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_BUILDING_BLOCK_TYPE]: 'default', + [ALERT_SEVERITY]: 'high', + [ALERT_RISK_SCORE]: 50, + [ALERT_RULE_PARAMETERS]: { + description: 'Detecting root and admin users', + risk_score: 50, + severity: 'high', + building_block_type: 'default', + note: '# Investigative notes', + license: 'Elastic License', + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + meta: { someMeta: 'someField' }, + author: ['Elastic'], + false_positives: [], + from: 'now-6m', + rule_id: 'rule-1', + max_signals: 10000, + risk_score_mapping: [], + severity_mapping: [], + threat: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0000', + name: 'test tactic', + reference: 'https://attack.mitre.org/tactics/TA0000/', + }, + technique: [ + { + id: 'T0000', + name: 'test technique', + reference: 'https://attack.mitre.org/techniques/T0000/', + subtechnique: [ + { + id: 'T0000.000', + name: 'test subtechnique', + reference: 'https://attack.mitre.org/techniques/T0000/000/', + }, + ], + }, + ], + }, + ], + to: 'now', + references: ['http://example.com', 'https://example.com'], + related_integrations: [], + required_fields: [], + setup: '', + version: 1, + exceptions_list: [ + { + id: 'some_uuid', + list_id: 'list_id_single', + namespace_type: 'single', + type: 'detection', + }, + { + id: 'endpoint_list', + list_id: 'endpoint_list', + namespace_type: 'agnostic', + type: 'endpoint', + }, + ], + immutable: false, + rule_source: { + type: 'internal', + }, + type: 'query', + language: 'kuery', + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + query: 'user.name: root or user.name: admin', + filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], + investigation_fields: undefined, + }, + [ALERT_RULE_INDICES]: completeRule.ruleParams.index, + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { + actions: [], + author: ['Elastic'], + uuid: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + building_block_type: 'default', + created_at: '2020-03-27T22:55:59.577Z', + updated_at: '2020-03-27T22:55:59.577Z', + created_by: 'sample user', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + immutable: false, + license: 'Elastic License', + meta: { + someMeta: 'someField', + }, + name: 'rule-name', + note: '# Investigative notes', + references: ['http://example.com', 'https://example.com'], + severity: 'high', + severity_mapping: [], + updated_by: 'sample user', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0000', + name: 'test tactic', + reference: 'https://attack.mitre.org/tactics/TA0000/', + }, + technique: [ + { + id: 'T0000', + name: 'test technique', + reference: 'https://attack.mitre.org/techniques/T0000/', + subtechnique: [ + { + id: 'T0000.000', + name: 'test subtechnique', + reference: 'https://attack.mitre.org/techniques/T0000/000/', + }, + ], + }, + ], + }, + ], + version: 1, + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + rule_id: 'rule-1', + interval: '5m', + exceptions_list: getListArrayMock(), + throttle: 'no_actions', + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + }), + [ALERT_DEPTH]: 1, + [ALERT_URL]: expectedAlertUrl, + [ALERT_UUID]: alertUuid, + [ALERT_WORKFLOW_TAGS]: [], + [ALERT_WORKFLOW_ASSIGNEE_IDS]: [], + someKey: 'someValue', + source: { + ip: '127.0.0.1', + }, + }; + expect(alert).toEqual(expected); + }); + + it('builds an alert as expected with original_event if present', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + [EVENT_ACTION]: 'socket_opened', + [EVENT_DATASET]: 'socket', + [EVENT_KIND]: 'event', + [EVENT_MODULE]: 'system', + }, + }; + const completeRule = getCompleteRuleMock(getQueryRuleParams()); + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + const expected = { + ...alert, + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }), + }; + expect(alert).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts new file mode 100644 index 0000000000000..c5112006ae251 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; +import { requiredOptional } from '@kbn/zod-helpers'; +import { EVENT_KIND } from '@kbn/rule-data-utils'; + +import type { BaseHit } from '../../../../../../common/detection_engine/types'; +import type { ConfigType } from '../../../../../config'; +import type { BuildReasonMessage } from '../../utils/reason_formatters'; +import { getMergeStrategy } from '../../utils/source_fields_merging/strategies'; +import type { SignalSource, SignalSourceHit } from '../../types'; +import { buildAlertFields, isThresholdResult } from './build_alert'; +import type { CompleteRule, RuleParams } from '../../../rule_schema'; +import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; +import { buildRuleNameFromMapping } from '../../utils/mappings/build_rule_name_from_mapping'; +import { buildSeverityFromMapping } from '../../utils/mappings/build_severity_from_mapping'; +import { buildRiskScoreFromMapping } from '../../utils/mappings/build_risk_score_from_mapping'; +import type { BaseFieldsLatest } from '../../../../../../common/api/detection_engine/model/alerts'; +import { traverseAndMutateDoc } from './traverse_and_mutate_doc'; +import { ALERT_THRESHOLD_RESULT } from '../../../../../../common/field_maps/field_names'; +import { robustGet, robustSet } from '../../utils/source_fields_merging/utils/robust_field_access'; + +const isSourceDoc = (hit: SignalSourceHit): hit is BaseHit => { + return hit._source != null && hit._id != null; +}; + +export interface TransformHitToAlertProps { + spaceId: string | null | undefined; + completeRule: CompleteRule; + doc: estypes.SearchHit; + mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: Record; + ignoreFieldsRegexes: string[]; + applyOverrides: boolean; + buildReasonMessage: BuildReasonMessage; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + alertUuid: string; + publicBaseUrl?: string; +} + +/** + * Formats the search_after result for insertion into the signals index. We first create a + * "best effort" merged "fields" with the "_source" object, then build the signal object, + * then the event object, and finally we strip away any additional temporary data that was added + * such as the "threshold_result". + * @param completeRule The rule saved object to build overrides + * @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result" + * @returns The body that can be added to a bulk call for inserting the signal. + */ +export const transformHitToAlert = ({ + spaceId, + completeRule, + doc, + mergeStrategy, + ignoreFields, + ignoreFieldsRegexes, + applyOverrides, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, +}: TransformHitToAlertProps): BaseFieldsLatest => { + const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields, ignoreFieldsRegexes }); + const thresholdResult = mergedDoc._source?.threshold_result; + + if (isSourceDoc(mergedDoc)) { + const overrides = applyOverrides + ? { + nameOverride: buildRuleNameFromMapping({ + eventSource: mergedDoc._source ?? {}, + ruleName: completeRule.ruleConfig.name, + ruleNameMapping: completeRule.ruleParams.ruleNameOverride, + }).ruleName, + severityOverride: buildSeverityFromMapping({ + eventSource: mergedDoc._source ?? {}, + severity: completeRule.ruleParams.severity, + severityMapping: completeRule.ruleParams.severityMapping, + }).severity, + riskScoreOverride: buildRiskScoreFromMapping({ + eventSource: mergedDoc._source ?? {}, + riskScore: completeRule.ruleParams.riskScore, + riskScoreMapping: requiredOptional(completeRule.ruleParams.riskScoreMapping), + }).riskScore, + } + : undefined; + + const reason = buildReasonMessage({ + name: overrides?.nameOverride ?? completeRule.ruleConfig.name, + severity: overrides?.severityOverride ?? completeRule.ruleParams.severity, + mergedDoc, + }); + + const alertFields = buildAlertFields({ + docs: [mergedDoc], + completeRule, + spaceId, + reason, + indicesToQuery, + alertUuid, + publicBaseUrl, + alertTimestampOverride, + overrides, + }); + + const { result: validatedSource, removed: removedSourceFields } = traverseAndMutateDoc( + mergedDoc._source + ); + + // The `alertFields` we add to alerts contain `event.kind: 'signal'` in dot notation. To avoid duplicating `event.kind`, + // we remove any existing `event.kind` field here before we merge `alertFields` into `validatedSource` later on + if (robustGet({ key: EVENT_KIND, document: validatedSource }) != null) { + robustSet({ key: EVENT_KIND, document: validatedSource, valueToSet: undefined }); + } + + if (removedSourceFields.length) { + ruleExecutionLogger?.debug( + 'Following fields were removed from alert source as ECS non-compliant:', + JSON.stringify(removedSourceFields) + ); + } + + merge(validatedSource, alertFields); + if (thresholdResult != null && isThresholdResult(thresholdResult)) { + validatedSource[ALERT_THRESHOLD_RESULT] = thresholdResult; + } + return validatedSource as BaseFieldsLatest; + } + + throw Error('Error building alert from source document.'); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.test.ts similarity index 63% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.test.ts index 21f9adc96bd60..0ec57843c83da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.test.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { stripNonEcsFields } from './strip_non_ecs_fields'; +import { traverseAndMutateDoc } from './traverse_and_mutate_doc'; -describe('stripNonEcsFields', () => { +describe('traverseAndMutateDoc', () => { it('should not strip ECS compliant fields', () => { const document = { client: { @@ -19,14 +19,14 @@ describe('stripNonEcsFields', () => { }, }; - const { result, removed } = stripNonEcsFields(document); + const { result, removed } = traverseAndMutateDoc(document); expect(result).toEqual(document); expect(removed).toEqual([]); }); it('should strip source object field if ECS mapping is not object', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: { name: { first: 'test-1', @@ -54,7 +54,7 @@ describe('stripNonEcsFields', () => { }); it('should strip source keyword field if ECS mapping is object', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: 'test', message: 'test message', }); @@ -74,7 +74,7 @@ describe('stripNonEcsFields', () => { // https://github.com/elastic/sdh-security-team/issues/736 describe('fields that exists in the alerts mapping but not in local ECS(ruleRegistry) definition', () => { it('should strip object type "device" field if it is supplied as a keyword', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ device: 'test', message: 'test message', }); @@ -94,7 +94,7 @@ describe('stripNonEcsFields', () => { describe('array fields', () => { it('should not strip arrays of objects when an object is expected', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: [{ name: 'agent-1' }, { name: 'agent-2' }], message: 'test message', }); @@ -107,7 +107,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting fields in array of objects', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: [ { name: 'agent-1', @@ -135,7 +135,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting array of keyword fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: ['agent-1', 'agent-2'], message: 'test message', }); @@ -156,7 +156,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting array of object fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: { name: [{ conflict: 'agent-1' }, { conflict: 'agent-2' }], type: 'filebeat' }, message: 'test message', }); @@ -176,11 +176,29 @@ describe('stripNonEcsFields', () => { }, ]); }); + + it('should entirely strip objects that end up empty in arrays', () => { + const { result, removed } = traverseAndMutateDoc({ + agent: [{ name: { conflict: 'agent-1' } }, { name: 'test' }], + message: 'test message', + }); + + expect(result).toEqual({ + agent: [{ name: 'test' }], + message: 'test message', + }); + expect(removed).toEqual([ + { + key: 'agent.name', + value: { conflict: 'agent-1' }, + }, + ]); + }); }); describe('dot notation', () => { it('should strip conflicting fields that use dot notation', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'agent.name.conflict': 'some-value', message: 'test message', }); @@ -198,7 +216,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting fields that use dot notation and is an array', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'agent.name.text': ['1'], message: 'test message', }); @@ -215,8 +233,8 @@ describe('stripNonEcsFields', () => { ]); }); - it('should strip conflicting fields that use dot notation and is an empty array', () => { - const { result, removed } = stripNonEcsFields({ + it('should strip conflicting fields that use dot notation and is an empty array but not report the empty array as removed', () => { + const { result, removed } = traverseAndMutateDoc({ 'agent.name.text': [], message: 'test message', }); @@ -225,16 +243,11 @@ describe('stripNonEcsFields', () => { message: 'test message', }); - expect(removed).toEqual([ - { - key: 'agent.name.text', - value: [], - }, - ]); + expect(removed).toEqual([]); }); it('should not strip valid ECS fields that use dot notation', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'agent.name': 'some name', 'agent.build.original': 'v10', message: 'test message', @@ -253,7 +266,7 @@ describe('stripNonEcsFields', () => { describe('non-ECS fields', () => { it('should not strip non-ECS fields that don`t conflict', () => { expect( - stripNonEcsFields({ + traverseAndMutateDoc({ non_ecs_object: { field1: 'value1', }, @@ -272,7 +285,7 @@ describe('stripNonEcsFields', () => { it('should not strip non-ECS fields that don`t conflict even when nested inside ECS fieldsets', () => { expect( - stripNonEcsFields({ + traverseAndMutateDoc({ agent: { non_ecs_object: { field1: 'value1', @@ -298,7 +311,7 @@ describe('stripNonEcsFields', () => { describe('ip field', () => { it('should not strip valid CIDR', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ source: { ip: '192.168.0.0', name: 'test source', @@ -315,7 +328,7 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid ip', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ source: { ip: 'invalid-ip', name: 'test source', @@ -336,7 +349,7 @@ describe('stripNonEcsFields', () => { describe('nested field', () => { it('should strip invalid nested', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ threat: { enrichments: ['non-valid-threat-1', 'non-valid-threat-2'], 'indicator.port': 443, @@ -361,7 +374,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid values', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ threat: { enrichments: [ { @@ -386,7 +399,7 @@ describe('stripNonEcsFields', () => { describe('date field', () => { it('should strip invalid date', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ event: { created: true, category: 'start', @@ -397,6 +410,7 @@ describe('stripNonEcsFields', () => { event: { category: 'start', }, + 'kibana.alert.original_event.category': 'start', }); expect(removed).toEqual([ { @@ -407,7 +421,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip string or number date field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ event: { created: '2020-12-12', end: [2345562, '2022-10-12'], @@ -419,6 +433,8 @@ describe('stripNonEcsFields', () => { created: '2020-12-12', end: [2345562, '2022-10-12'], }, + 'kibana.alert.original_event.created': '2020-12-12', + 'kibana.alert.original_event.end': [2345562, '2022-10-12'], }); expect(removed).toEqual([]); }); @@ -426,13 +442,13 @@ describe('stripNonEcsFields', () => { describe('long field', () => { it('should strip invalid long field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ client: { bytes: 'non-valid', }, }); - expect(result).toEqual({ client: {} }); + expect(result).toEqual({}); expect(removed).toEqual([ { key: 'client.bytes', @@ -442,13 +458,13 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid long field with space in it', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ client: { bytes: '24 ', }, }); - expect(result).toEqual({ client: {} }); + expect(result).toEqual({}); expect(removed).toEqual([ { key: 'client.bytes', @@ -459,7 +475,7 @@ describe('stripNonEcsFields', () => { }); describe('numeric field', () => { it('should strip invalid float field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'user.risk.calculated_score': 'non-valid', }); @@ -473,13 +489,13 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid scaled_float field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ host: { 'cpu.usage': 'non-valid', }, }); - expect(result).toEqual({ host: {} }); + expect(result).toEqual({}); expect(removed).toEqual([ { key: 'host.cpu.usage', @@ -489,7 +505,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip string float field with space', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'user.risk.calculated_score': '24 ', }); @@ -500,7 +516,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip string scaled_float field with space', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'host.cpu.usage': '24 ', }); @@ -511,7 +527,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid number in string field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ host: { 'cpu.usage': '1234', }, @@ -526,7 +542,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip array of valid numeric fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'user.risk.calculated_score': [458.3333, '45.3', 10, 0, -667.23], }); @@ -539,7 +555,7 @@ describe('stripNonEcsFields', () => { describe('boolean field', () => { it('should strip invalid boolean fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': ['conflict', 'true', 5, 'False', 'ee', 'True'], }); @@ -571,7 +587,7 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid boolean True', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': 'True', }); @@ -585,7 +601,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid boolean fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': ['true', 'false', true, false, ''], }); @@ -596,7 +612,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid boolean fields nested in array', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': [[true, false], ''], }); @@ -610,7 +626,7 @@ describe('stripNonEcsFields', () => { // geo_point is too complex so we going to skip its validation describe('geo_point field', () => { it('should not strip invalid geo_point field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'client.location.geo': 'invalid geo_point', }); @@ -622,7 +638,7 @@ describe('stripNonEcsFields', () => { it('should not strip valid geo_point fields', () => { expect( - stripNonEcsFields({ + traverseAndMutateDoc({ 'client.geo.location': [0, 90], }).result ).toEqual({ @@ -630,7 +646,7 @@ describe('stripNonEcsFields', () => { }); expect( - stripNonEcsFields({ + traverseAndMutateDoc({ 'client.geo.location': { type: 'Point', coordinates: [-88.34, 20.12], @@ -644,7 +660,7 @@ describe('stripNonEcsFields', () => { }); expect( - stripNonEcsFields({ + traverseAndMutateDoc({ 'client.geo.location': 'POINT (-71.34 41.12)', }).result ).toEqual({ @@ -652,7 +668,7 @@ describe('stripNonEcsFields', () => { }); expect( - stripNonEcsFields({ + traverseAndMutateDoc({ client: { geo: { location: { @@ -674,4 +690,152 @@ describe('stripNonEcsFields', () => { }); }); }); + + describe('globally ignored fields', () => { + it('should strip out globally ignored top level fields', () => { + const { result, removed } = traverseAndMutateDoc({ + kibana: 'test-value', + non_ecs_field: 'value', + }); + + expect(result).toEqual({ + non_ecs_field: 'value', + }); + expect(removed).toEqual([ + { + key: 'kibana', + value: 'test-value', + }, + ]); + }); + + it('should strip out globally ignored nested fields', () => { + const { result, removed } = traverseAndMutateDoc({ + 'kibana.test': 'test-value', + non_ecs_field: 'value', + }); + + expect(result).toEqual({ + non_ecs_field: 'value', + }); + expect(removed).toEqual([ + { + key: 'kibana.test', + value: 'test-value', + }, + ]); + }); + + it('should not strip out fields that use ignored field names as a prefix', () => { + const { result, removed } = traverseAndMutateDoc({ + kibana_test_prefix: 'test-value', + non_ecs_field: 'value', + }); + + expect(result).toEqual({ + kibana_test_prefix: 'test-value', + non_ecs_field: 'value', + }); + expect(removed).toEqual([]); + }); + }); + + describe('fieldsToAdd', () => { + it('should extract a nested event field to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ event: { action: 'test' } }); + expect(result).toEqual({ + event: { action: 'test' }, + 'kibana.alert.original_event.action': 'test', + }); + expect(removed).toEqual([]); + }); + + it('should extract multiple nested event fields to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + event: { action: 'test', field2: 'test2' }, + }); + expect(result).toEqual({ + event: { action: 'test', field2: 'test2' }, + 'kibana.alert.original_event.action': 'test', + 'kibana.alert.original_event.field2': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should extract a dot notation event field to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ 'event.action': 'test' }); + expect(result).toEqual({ + 'event.action': 'test', + 'kibana.alert.original_event.action': 'test', + }); + expect(removed).toEqual([]); + }); + + it('should extract multiple dot notation event fields to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + 'event.action': 'test', + 'event.field2': 'test2', + }); + expect(result).toEqual({ + 'event.action': 'test', + 'event.field2': 'test2', + 'kibana.alert.original_event.action': 'test', + 'kibana.alert.original_event.field2': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should extract mixed notation fields to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + event: { 'field.subfield': 'test', 'field2.subfield': 'test2' }, + }); + expect(result).toEqual({ + event: { 'field.subfield': 'test', 'field2.subfield': 'test2' }, + 'kibana.alert.original_event.field.subfield': 'test', + 'kibana.alert.original_event.field2.subfield': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should extract mixed notation with dot notation first to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + 'event.field': { subfield: 'test', subfield2: 'test2' }, + }); + expect(result).toEqual({ + 'event.field': { subfield: 'test', subfield2: 'test2' }, + 'kibana.alert.original_event.field.subfield': 'test', + 'kibana.alert.original_event.field.subfield2': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should not extract original event fields if they are not top level', () => { + const { result, removed } = traverseAndMutateDoc({ + 'top_field.event.action': 'test', + }); + expect(result).toEqual({ 'top_field.event.action': 'test' }); + expect(removed).toEqual([]); + }); + + it('should not duplicate added fields', () => { + const { result, removed } = traverseAndMutateDoc({ event: { event: 'test' } }); + expect(result).toEqual({ + event: { event: 'test' }, + 'kibana.alert.original_event.event': 'test', + }); + expect(removed).toEqual([]); + }); + + it('should work on multiple levels of nesting', () => { + const { result, removed } = traverseAndMutateDoc({ + event: { field: { subfield: 'test', subfield2: 'test2' } }, + }); + expect(result).toEqual({ + event: { field: { subfield: 'test', subfield2: 'test2' } }, + 'kibana.alert.original_event.field.subfield': 'test', + 'kibana.alert.original_event.field.subfield2': 'test2', + }); + expect(removed).toEqual([]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.ts similarity index 50% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.ts index 08e5fa5fd879d..c9720a139ae7d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.ts @@ -6,8 +6,9 @@ */ import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; -import { isPlainObject, cloneDeep, isArray } from 'lodash'; +import { isPlainObject, isArray, set } from 'lodash'; import type { SearchTypes } from '../../../../../../common/detection_engine/types'; import { isValidIpType } from './ecs_types_validators/is_valid_ip_type'; @@ -15,6 +16,10 @@ import { isValidDateType } from './ecs_types_validators/is_valid_date_type'; import { isValidNumericType } from './ecs_types_validators/is_valid_numeric_type'; import { isValidBooleanType } from './ecs_types_validators/is_valid_boolean_type'; import { isValidLongType } from './ecs_types_validators/is_valid_long_type'; +import { + ALERT_ORIGINAL_EVENT, + ALERT_THRESHOLD_RESULT, +} from '../../../../../../common/field_maps/field_names'; type SourceFieldRecord = Record; type SourceField = SearchTypes | SourceFieldRecord; @@ -153,67 +158,149 @@ const computeIsEcsCompliant = (value: SourceField, path: string) => { return isEcsFieldObject ? isPlainObject(value) : !isPlainObject(value); }; -interface StripNonEcsFieldsReturn { - result: SourceFieldRecord; - removed: Array<{ key: string; value: SearchTypes }>; -} +const bannedFields = ['kibana', 'signal', 'threshold_result', ALERT_THRESHOLD_RESULT]; /** - * strips alert source object from ECS non compliant fields + * Traverse an entire source document and mutate it to prepare for indexing into the alerts index. Traversing the document + * is computationally expensive so we only want to traverse it once, therefore a few distinct cases are handled in this function: + * 1. Fields that we must explicitly remove, like `kibana` and `signal`, fields, are removed from the document. + * 2. Fields that are incompatible with ECS are removed. + * 3. All `event.*` fields are collected and copied to `kibana.alert.original_event.*` using `fieldsToAdd` + * @param document The document to traverse + * @returns The mutated document, a list of removed fields */ -export const stripNonEcsFields = (doc: SourceFieldRecord): StripNonEcsFieldsReturn => { - const result = cloneDeep(doc); - const removed: Array<{ key: string; value: SearchTypes }> = []; - - /** - * traverses through object and deletes ECS non compliant fields - * @param document - document to traverse - * @param documentKey - document key in parent document, if exists - * @param parent - parent of traversing document - * @param parentPath - path of parent in initial source document - */ - const traverseAndDeleteInObj = ( - document: SourceField, - documentKey: string, - parent?: SourceFieldRecord, - parentPath?: string - ) => { - const fullPath = [parentPath, documentKey].filter(Boolean).join('.'); - // if document array, traverse through each item w/o changing documentKey, parent, parentPath - if (isArray(document) && document.length > 0) { - document.slice().forEach((value) => { - traverseAndDeleteInObj(value, documentKey, parent, parentPath); +export const traverseAndMutateDoc = (document: SourceFieldRecord) => { + const { result, removed, fieldsToAdd } = internalTraverseAndMutateDoc({ + document, + path: [], + topLevel: true, + removed: [], + fieldsToAdd: [], + }); + + fieldsToAdd.forEach(({ key, value }) => { + result[key] = value; + }); + + return { result, removed }; +}; + +const internalTraverseAndMutateDoc = ({ + document, + path, + topLevel, + removed, + fieldsToAdd, +}: { + document: T; + path: string[]; + topLevel: boolean; + removed: Array<{ key: string; value: SearchTypes }>; + fieldsToAdd: Array<{ key: string; value: SearchTypes }>; +}) => { + Object.keys(document).forEach((key) => { + // Using Object.keys and fetching the value for each key separately performs better in profiling than using Object.entries + const value = document[key]; + const fullPathArray = [...path, key]; + const fullPath = fullPathArray.join('.'); + // Insert checks that don't care about the value - only depend on the key - up here + let deleted = false; + if (topLevel) { + const firstKeyString = key.split('.')[0]; + bannedFields.forEach((bannedField) => { + if (firstKeyString === bannedField) { + delete document[key]; + deleted = true; + removed.push({ key: fullPath, value }); + } }); - return; } - if (parent && !computeIsEcsCompliant(document, fullPath)) { - const documentReference = parent[documentKey]; - // if document reference in parent is array, remove only this item from array - // e.g. a boolean mapped field with values ['not-boolean', 'true'] should strip 'not-boolean' and leave 'true' - if (isArray(documentReference)) { - const indexToDelete = documentReference.findIndex((item) => item === document); - documentReference.splice(indexToDelete, 1); - if (documentReference.length === 0) { - delete parent[documentKey]; + // If we passed the key check, additional checks based on key and value are done below. Items in arrays are treated independently from each other. + if (!deleted) { + if (isArray(value)) { + const newValue = traverseArray({ array: value, path: fullPathArray, removed, fieldsToAdd }); + if (newValue.length > 0) { + set(document, key, newValue); + } else { + delete document[key]; + deleted = true; + } + } else if (!computeIsEcsCompliant(value, fullPath)) { + delete document[key]; + deleted = true; + removed.push({ key: fullPath, value }); + } else if (isSearchTypesRecord(value)) { + internalTraverseAndMutateDoc({ + document: value, + path: fullPathArray, + topLevel: false, + removed, + fieldsToAdd, + }); + if (Object.keys(value).length === 0) { + delete document[key]; + deleted = true; + } + } + } + + // We're keeping the field, but maybe we want to copy it to a different field as well + if (!deleted && fullPath.split('.')[0] === 'event' && topLevel) { + // The value might have changed above when we `set` after traversing an array + const valueRefetch = document[key]; + const newKey = `${ALERT_ORIGINAL_EVENT}${fullPath.replace('event', '')}`; + if (isPlainObject(valueRefetch)) { + const flattenedObject = flattenWithPrefix(newKey, valueRefetch); + for (const [k, v] of Object.entries(flattenedObject)) { + fieldsToAdd.push({ key: k, value: v }); } } else { - delete parent[documentKey]; + fieldsToAdd.push({ + key: `${ALERT_ORIGINAL_EVENT}${fullPath.replace('event', '')}`, + value: valueRefetch, + }); } - removed.push({ key: fullPath, value: document }); - return; } + }); + return { result: document, removed, fieldsToAdd }; +}; - if (isSearchTypesRecord(document)) { - Object.entries(document).forEach(([key, value]) => { - traverseAndDeleteInObj(value, key, document, fullPath); +const traverseArray = ({ + array, + path, + removed, + fieldsToAdd, +}: { + array: SearchTypes[]; + path: string[]; + removed: Array<{ key: string; value: SearchTypes }>; + fieldsToAdd: Array<{ key: string; value: SearchTypes }>; +}): SearchTypes[] => { + const pathString = path.join('.'); + for (let i = 0; i < array.length; i++) { + const value = array[i]; + if (isArray(value)) { + array[i] = traverseArray({ array: value, path, removed, fieldsToAdd }); + } + } + return array.filter((value) => { + if (isArray(value)) { + return value.length > 0; + } else if (!computeIsEcsCompliant(value, pathString)) { + removed.push({ key: pathString, value }); + return false; + } else if (isSearchTypesRecord(value)) { + internalTraverseAndMutateDoc({ + document: value, + path, + topLevel: false, + removed, + fieldsToAdd, }); + return Object.keys(value).length > 0; + } else { + return true; } - }; - - traverseAndDeleteInObj(result, ''); - return { - result, - removed, - }; + }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 6b21ed226c165..2495e48c1cc81 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -10,7 +10,7 @@ import type { ConfigType } from '../../../../config'; import type { SignalSource, SimpleHit } from '../types'; import type { CompleteRule, RuleParams } from '../../rule_schema'; import { generateId } from '../utils/utils'; -import { buildBulkBody } from './utils/build_bulk_body'; +import { transformHitToAlert } from './utils/transform_hit_to_alert'; import type { BuildReasonMessage } from '../utils/reason_formatters'; import type { BaseFieldsLatest, @@ -22,6 +22,7 @@ export const wrapHitsFactory = ({ completeRule, ignoreFields, + ignoreFieldsRegexes, mergeStrategy, spaceId, indicesToQuery, @@ -30,7 +31,8 @@ export const wrapHitsFactory = ruleExecutionLogger, }: { completeRule: CompleteRule; - ignoreFields: ConfigType['alertIgnoreFields']; + ignoreFields: Record; + ignoreFieldsRegexes: string[]; mergeStrategy: ConfigType['alertMergeStrategy']; spaceId: string | null | undefined; indicesToQuery: string[]; @@ -51,20 +53,21 @@ export const wrapHitsFactory = `${spaceId}:${completeRule.alertId}` ); - const baseAlert = buildBulkBody( + const baseAlert = transformHitToAlert({ spaceId, completeRule, - event as SimpleHit, + doc: event as SimpleHit, mergeStrategy, ignoreFields, - true, + ignoreFieldsRegexes, + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts index 5028c15c2c8d1..efd33c5442bf3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts @@ -18,7 +18,7 @@ import type { CompleteRule, RuleParams } from '../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { SignalSource } from '../types'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; export interface EventsAndTerms { event: estypes.SearchHit; @@ -52,20 +52,21 @@ export const wrapNewTermsAlerts = ({ `${spaceId}:${completeRule.alertId}`, eventAndTerms.newTerms, ]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - eventAndTerms.event, + doc: eventAndTerms.event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts index 274a11fc4ffcc..ad34feb81eab1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts @@ -21,7 +21,7 @@ import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import { getSuppressionAlertFields, getSuppressionTerms } from '../utils'; import type { SignalSource } from '../types'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; export interface EventsAndTerms { event: estypes.SearchHit; @@ -69,20 +69,21 @@ export const wrapSuppressedNewTermsAlerts = ({ eventAndTerms.newTerms, ]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts index e7bfe3f7eaacd..abff900a5dcb4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts @@ -23,7 +23,7 @@ import type { ConfigType } from '../../../../../config'; import type { CompleteRule, RuleParams } from '../../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; import type { SignalSource } from '../../types'; -import { buildBulkBody } from '../../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../../factories/utils/transform_hit_to_alert'; import type { BuildReasonMessage } from '../../utils/reason_formatters'; export interface SuppressionBucket { @@ -82,20 +82,21 @@ export const wrapSuppressedAlerts = ({ ruleId: completeRule.alertId, spaceId, }); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - bucket.event, + doc: bucket.event, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts index 492d27fba091f..e9acb8284b740 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts @@ -25,7 +25,7 @@ import type { import type { ConfigType } from '../../../../config'; import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { ThresholdBucket } from './types'; import type { BuildReasonMessage } from '../utils/reason_formatters'; @@ -86,20 +86,21 @@ export const wrapSuppressedThresholdALerts = ({ const instanceId = objectHash([suppressedValues, completeRule.alertId, spaceId]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - hit, + doc: hit, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts index 4d414d71cfadf..60b7e3edacedf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts @@ -10,7 +10,6 @@ import { repeatedSearchResultsWithSortId, repeatedSearchResultsWithNoSortId, sampleDocSearchResultsNoSortIdNoHits, - sampleDocWithSortId, } from '../__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; @@ -114,7 +113,8 @@ describe('searchAfterAndBulkCreate', () => { wrapHits = wrapHitsFactory({ completeRule: queryCompleteRule, mergeStrategy: 'missingFields', - ignoreFields: [], + ignoreFields: {}, + ignoreFieldsRegexes: [], spaceId: 'default', indicesToQuery: inputIndexPattern, alertTimestampOverride: undefined, @@ -1025,8 +1025,22 @@ describe('searchAfterAndBulkCreate', () => { expect(mockEnrichment).toHaveBeenCalledWith( expect.objectContaining([ expect.objectContaining({ - ...sampleDocWithSortId(), _id: expect.any(String), + _index: 'myFakeSignalIndex', + _score: 100, + _source: expect.objectContaining({ + destination: { ip: '127.0.0.1' }, + someKey: 'someValue', + source: { ip: '127.0.0.1' }, + }), + _version: 1, + fields: { + '@timestamp': ['2020-04-20T21:27:45+0000'], + 'destination.ip': ['127.0.0.1'], + someKey: ['someValue'], + 'source.ip': ['127.0.0.1'], + }, + sort: ['1234567891111', '2233447556677'], }), ]) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts index 2b69f4fe980f2..6f4f15ad942be 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts @@ -46,7 +46,11 @@ describe('merge_all_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -55,7 +59,11 @@ describe('merge_all_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -64,7 +72,11 @@ describe('merge_all_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -73,7 +85,11 @@ describe('merge_all_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -82,7 +98,11 @@ describe('merge_all_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -91,7 +111,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -100,7 +124,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -109,7 +137,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -135,7 +167,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -144,7 +180,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -153,7 +193,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -162,7 +206,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -171,7 +219,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -180,7 +232,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -189,7 +245,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -219,7 +279,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -228,7 +292,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -237,7 +305,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -246,7 +318,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -255,7 +331,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -264,7 +344,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -273,7 +357,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -301,7 +389,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -310,7 +402,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -319,7 +415,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -328,7 +428,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -337,7 +441,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -346,7 +454,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -355,7 +467,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -378,7 +494,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -391,7 +511,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -404,7 +528,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -415,7 +543,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -442,7 +574,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -455,7 +591,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -466,7 +606,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -479,7 +623,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }); @@ -505,7 +653,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': 'other_value_1', }); @@ -516,7 +668,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -525,7 +681,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': { zed: 'other_value_1' }, }); @@ -536,7 +696,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -562,7 +726,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -573,7 +741,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -584,7 +756,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -595,7 +771,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -621,7 +801,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -630,7 +814,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -639,7 +827,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -648,7 +840,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -672,7 +868,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -683,7 +883,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -694,7 +898,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -705,7 +913,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -731,7 +943,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -740,7 +956,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -749,7 +969,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -758,7 +982,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -784,7 +1012,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'], @@ -797,7 +1029,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -810,7 +1046,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }], @@ -823,7 +1063,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], @@ -851,7 +1095,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -860,7 +1108,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -869,7 +1121,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -878,7 +1134,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -904,7 +1164,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -913,7 +1177,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -922,7 +1190,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -933,7 +1205,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -959,7 +1235,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -968,7 +1248,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -977,7 +1261,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': { mars: 'other_value_1' }, }); @@ -988,7 +1276,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }); @@ -1016,7 +1308,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1025,7 +1321,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1034,7 +1334,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1045,7 +1349,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1071,7 +1379,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1080,7 +1392,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1089,7 +1405,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1100,7 +1420,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1126,7 +1450,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1135,7 +1463,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1144,7 +1476,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -1153,7 +1489,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -1177,8 +1517,12 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; - expect(merged).toEqual(_source); + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; + expect(merged).toEqual(cloneDeep(_source)); }); test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_[{}1, ...1])"', () => { @@ -1186,8 +1530,12 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; - expect(merged).toEqual(_source); + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; + expect(merged).toEqual(cloneDeep(_source)); }); test('fields has a single nested object (f_[{}2]), merged doc is array value (f_[{}2])"', () => { @@ -1195,7 +1543,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -1204,7 +1556,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -1230,10 +1586,14 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - foo: { bar: 'value_1' }, - 'foo.bar': 'other_value_1', + foo: { bar: 'other_value_1' }, + 'foo.bar': 'value_2', }); }); @@ -1245,10 +1605,14 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - foo: { bar: 'value_1' }, // <--- We have duplicated value_1 twice which is a bug - 'foo.bar': ['value_1', 'value_2'], // <-- We have merged the array value because we do not understand if we should or not + foo: { bar: ['value_1', 'value_2'] }, // <-- We have merged the array value because we do not understand if we should or not + 'foo.bar': 'value_2', // <--- We have duplicated value_2 twice which is a bug }); }); }); @@ -1272,7 +1636,11 @@ describe('merge_all_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'foo_other_value_1', bar: 'bar_other_value_1', @@ -1293,7 +1661,11 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ host: { hostname: 'hostname_other_value_1', @@ -1318,7 +1690,11 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { host: { @@ -1339,7 +1715,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1353,7 +1733,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1365,7 +1749,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1385,7 +1773,11 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'host.name': 'host_name_other_value_1', 'host.hostname': 'hostname_other_value_1', @@ -1404,7 +1796,11 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.host.name': 'host_name_other_value_1', 'foo.host.hostname': 'hostname_other_value_1', @@ -1419,7 +1815,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1434,7 +1834,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1447,7 +1851,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1472,7 +1880,11 @@ describe('merge_all_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1492,7 +1904,11 @@ describe('merge_all_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1507,7 +1923,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'single_value', zed: 'single_value' }, }); @@ -1526,7 +1946,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: [{ bar: ['single_value'], zed: ['single_value'] }], }); @@ -1543,15 +1967,15 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - email: { - headers: { - 'x-test': 'from fields', - }, + 'email.headers': { + 'x-test': 'from fields', }, - // preserves conflicting keys if values contain empty objects - 'email.headers': {}, }); }); @@ -1564,7 +1988,11 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': 'from fields', @@ -1581,7 +2009,11 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': 'from fields' }, }); @@ -1596,7 +2028,11 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['b1', 'b2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': ['b1', 'b2'] }, }); @@ -1611,7 +2047,11 @@ describe('merge_all_fields_with_source', () => { 'a.b.c.d': ['5', '6'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1624,7 +2064,11 @@ describe('merge_all_fields_with_source', () => { 'a.b.c': [{ d: '3 ' }, { d: '4' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'a.b': { c: [{ d: '3 ' }, { d: '4' }] }, }); @@ -1645,7 +2089,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['value.should.ignore', '/[_]+/'], + ignoreFields: { 'value.should.ignore': true }, + ignoreFieldsRegexes: ['/[_]+/'], })._source; expect(merged).toEqual({ foo: { @@ -1664,7 +2109,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + ignoreFields: { 'other.string': true }, + ignoreFieldsRegexes: ['/[z]+/'], // Neither of these two should match anything })._source; expect(merged).toEqual({ foo: { @@ -1694,7 +2140,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['value.should.ignore', '/[_]+/'], + ignoreFields: { 'value.should.ignore': true }, + ignoreFieldsRegexes: ['/[_]+/'], })._source; expect(merged).toEqual({ foo: { @@ -1718,7 +2165,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['nothing.to.match', '/[z]+/'], // these match nothing + ignoreFields: { 'nothing.to.match': true }, + ignoreFieldsRegexes: ['/[z]+/'], // these match nothing })._source; expect(merged).toEqual({ foo: { @@ -1743,7 +2191,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: [], + ignoreFields: {}, + ignoreFieldsRegexes: [], })._source; expect(merged).toEqual({ foo: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts index 762542d2b93dc..7b75899fe60f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts @@ -5,10 +5,9 @@ * 2.0. */ -import { get } from 'lodash/fp'; -import { set } from '@kbn/safer-lodash-set/fp'; +import { robustGet, robustSet } from '../utils/robust_field_access'; import type { SignalSource } from '../../../types'; -import { filterFieldEntries } from '../utils/filter_field_entries'; +import { filterFieldEntry } from '../utils/filter_field_entry'; import type { FieldsType, MergeStrategyFunction } from '../types'; import { isObjectLikeOrArrayOfObjectLikes } from '../utils/is_objectlike_or_array_of_objectlikes'; import { isNestedObject } from '../utils/is_nested_object'; @@ -16,8 +15,7 @@ import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; import { isPrimitive } from '../utils/is_primitive'; import { isArrayOfPrimitives } from '../utils/is_array_of_primitives'; import { isTypeObject } from '../utils/is_type_object'; -import { isPathValid } from '../utils/is_path_valid'; -import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map'; +import { robustIsPathValid } from '../utils/is_path_valid'; /** * Merges all of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information @@ -29,63 +27,56 @@ import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { +export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ + doc, + ignoreFields, + ignoreFieldsRegexes, +}) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; - const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); - const fieldsKeyMap = buildFieldsKeyAsArrayMap(source); - - const transformedSource = filteredEntries.reduce( - (merged, [fieldsKeyAsString, fieldsValue]: [string, FieldsType]) => { - const fieldsKey = fieldsKeyMap[fieldsKeyAsString] ?? fieldsKeyAsString; - - if ( - hasEarlyReturnConditions({ - fieldsValue, - fieldsKey, - merged, - }) - ) { - return merged; - } + const fieldsKeys = Object.keys(fields); - const valueInMergedDocument = get(fieldsKey, merged); + fieldsKeys.forEach((fieldsKey) => { + const valueInMergedDocument = robustGet({ key: fieldsKey, document: source }); + const fieldsValue = fields[fieldsKey]; + if ( + !hasEarlyReturnConditions({ + fieldsValue, + fieldsKey, + merged: source, + }) && + filterFieldEntry([fieldsKey, fieldsValue], fieldsKeys, ignoreFields, ignoreFieldsRegexes) + ) { if (valueInMergedDocument === undefined) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if (isPrimitive(valueInMergedDocument)) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if (isArrayOfPrimitives(valueInMergedDocument)) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if ( isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && isNestedObject(fieldsValue) && !Array.isArray(valueInMergedDocument) ) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if ( isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && isNestedObject(fieldsValue) && Array.isArray(valueInMergedDocument) ) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); - } else { - // fail safe catch all condition for production, but we shouldn't try to reach here and - // instead write tests if we encounter this situation. - return merged; + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } - }, - { ...source } - ); + } + }); return { ...doc, - _source: transformedSource, + _source: source, }; }; @@ -105,13 +96,13 @@ const hasEarlyReturnConditions = ({ merged, }: { fieldsValue: FieldsType; - fieldsKey: string[] | string; + fieldsKey: string; merged: SignalSource; }) => { - const valueInMergedDocument = get(fieldsKey, merged); + const valueInMergedDocument = robustGet({ key: fieldsKey, document: merged }); return ( fieldsValue.length === 0 || - (valueInMergedDocument === undefined && !isPathValid(fieldsKey, merged)) || + (valueInMergedDocument === undefined && !robustIsPathValid(fieldsKey, merged)) || (isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && !isNestedObject(fieldsValue) && !isTypeObject(fieldsValue)) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts index 80b9360541563..04089f449c368 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts @@ -45,7 +45,11 @@ describe('merge_missing_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -54,7 +58,11 @@ describe('merge_missing_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -63,7 +71,11 @@ describe('merge_missing_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -72,7 +84,11 @@ describe('merge_missing_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -81,7 +97,11 @@ describe('merge_missing_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -90,7 +110,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -99,7 +123,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -108,7 +136,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -134,7 +166,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -143,7 +179,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -152,7 +192,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -161,7 +205,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -170,7 +218,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -179,7 +231,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -188,7 +244,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -218,7 +278,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -227,7 +291,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -236,7 +304,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -245,7 +317,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -254,7 +330,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -263,7 +343,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -272,7 +356,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -310,7 +398,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -319,7 +411,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -328,7 +424,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -337,7 +437,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -346,7 +450,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -355,7 +463,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -364,7 +476,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -390,7 +506,7 @@ describe('merge_missing_fields_with_source', () => { const start = performance.now(); // we don't care about the response just determining performance // eslint-disable-next-line @typescript-eslint/no-unused-expressions - mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + mergeMissingFieldsWithSource({ doc, ignoreFields: {}, ignoreFieldsRegexes: [] })._source; const end = performance.now(); expect(end - start).toBeLessThan(500); }); @@ -414,7 +530,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -427,7 +547,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -440,7 +564,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({}); }); @@ -449,7 +577,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({}); }); }); @@ -474,7 +606,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -483,7 +619,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -492,7 +632,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -501,7 +645,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -525,7 +673,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -534,7 +686,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -543,7 +699,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -552,7 +712,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -578,7 +742,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -587,7 +755,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -596,7 +768,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -605,7 +781,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -629,7 +809,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -638,7 +822,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -647,7 +835,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -656,7 +848,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -680,7 +876,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -689,7 +889,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -698,7 +902,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -707,7 +915,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -731,7 +943,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -740,7 +956,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -749,7 +969,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -758,7 +982,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -784,7 +1012,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -793,7 +1025,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -802,7 +1038,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -811,7 +1051,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -835,7 +1079,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -844,7 +1092,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -853,7 +1105,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -862,7 +1118,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -888,7 +1148,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -897,7 +1161,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -906,7 +1174,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -915,7 +1187,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -939,7 +1215,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -948,7 +1228,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -957,7 +1241,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -966,7 +1254,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -992,7 +1284,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1001,7 +1297,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1010,7 +1310,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1019,7 +1323,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1043,7 +1351,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1052,7 +1364,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1061,7 +1377,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1070,7 +1390,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1094,7 +1418,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1103,7 +1431,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1112,7 +1444,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1121,7 +1457,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1145,7 +1485,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1154,7 +1498,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1163,7 +1511,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1172,7 +1524,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1198,7 +1554,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1210,7 +1570,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1234,7 +1598,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1252,7 +1620,11 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1272,7 +1644,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1283,7 +1659,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1301,7 +1681,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1317,7 +1701,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1335,7 +1723,11 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1351,7 +1743,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1363,7 +1759,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1380,7 +1780,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1395,7 +1799,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1410,15 +1818,15 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - email: { - headers: { - 'x-test': 'from fields', - }, + 'email.headers': { + 'x-test': 'from fields', }, - // preserves conflicting keys if values contain empty objects - 'email.headers': {}, }); }); @@ -1431,7 +1839,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': 'from fields', @@ -1448,7 +1860,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1461,7 +1877,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1474,7 +1894,11 @@ describe('merge_missing_fields_with_source', () => { 'a.b.c.d': ['1', '2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1487,7 +1911,11 @@ describe('merge_missing_fields_with_source', () => { 'a.b.c': [{ d: '3 ' }, { d: '4' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1500,7 +1928,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['a'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1525,7 +1957,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1545,7 +1981,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1560,7 +2000,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({}); }); @@ -1577,7 +2021,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1596,7 +2044,8 @@ describe('merge_missing_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeMissingFieldsWithSource({ doc, - ignoreFields: ['value.should.ignore', '/[_]+/'], + ignoreFields: { 'value.should.ignore': true }, + ignoreFieldsRegexes: ['/[_]+/'], })._source; expect(merged).toEqual({ foo: { @@ -1615,7 +2064,8 @@ describe('merge_missing_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeMissingFieldsWithSource({ doc, - ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + ignoreFields: { 'other.string': true }, + ignoreFieldsRegexes: ['/[z]+/'], // Neither of these two should match anything })._source; expect(merged).toEqual({ foo: { @@ -1646,7 +2096,8 @@ describe('merge_missing_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeMissingFieldsWithSource({ doc, - ignoreFields: [], + ignoreFields: {}, + ignoreFieldsRegexes: [], })._source; expect(merged).toEqual({ foo: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts index b4de7a9d4bb1a..ad9d8bd34705b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts @@ -5,17 +5,13 @@ * 2.0. */ -import { get } from 'lodash/fp'; -import { set } from '@kbn/safer-lodash-set'; - -import type { SignalSource } from '../../../types'; -import { filterFieldEntries } from '../utils/filter_field_entries'; +import { filterFieldEntry } from '../utils/filter_field_entry'; import type { FieldsType, MergeStrategyFunction } from '../types'; import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; import { isTypeObject } from '../utils/is_type_object'; import { isNestedObject } from '../utils/is_nested_object'; -import { isPathValid } from '../utils/is_path_valid'; -import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map'; +import { robustGet, robustSet } from '../utils/robust_field_access'; +import { robustIsPathValid } from '../utils/is_path_valid'; /** * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information @@ -26,36 +22,33 @@ import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { +export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ + doc, + ignoreFields, + ignoreFieldsRegexes, +}) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; - const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); - const fieldsKeyMap = buildFieldsKeyAsArrayMap(source); + const fieldKeys = Object.keys(fields); + + fieldKeys.forEach((fieldKey) => { + const valueInMergedDocument = robustGet({ key: fieldKey, document: source }); + if (valueInMergedDocument == null && robustIsPathValid(fieldKey, source)) { + const fieldsValue = fields[fieldKey]; - const transformedSource = filteredEntries.reduce( - (merged, [fieldsKeyAsString, fieldsValue]: [string, FieldsType]) => { - const fieldsKey = fieldsKeyMap[fieldsKeyAsString] ?? fieldsKeyAsString; if ( - hasEarlyReturnConditions({ - fieldsValue, - fieldsKey, - merged, - }) + !hasEarlyReturnConditions(fieldsValue) && + filterFieldEntry([fieldKey, fieldsValue], fieldKeys, ignoreFields, ignoreFieldsRegexes) ) { - return merged; + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return robustSet({ document: source, key: fieldKey, valueToSet: valueToMerge }); } - - const valueInMergedDocument = get(fieldsKey, merged); - const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(merged, fieldsKey, valueToMerge); - }, - { ...source } - ); + } + }); return { ...doc, - _source: transformedSource, + _source: source, }; }; @@ -70,21 +63,6 @@ export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc, ignor * @param merged The merge document which is what we are testing conditions against * @returns true if we should return early, otherwise false */ -const hasEarlyReturnConditions = ({ - fieldsValue, - fieldsKey, - merged, -}: { - fieldsValue: FieldsType; - fieldsKey: string[] | string; - merged: SignalSource; -}) => { - const valueInMergedDocument = get(fieldsKey, merged); - return ( - fieldsValue.length === 0 || - valueInMergedDocument !== undefined || - !isPathValid(fieldsKey, merged) || - isNestedObject(fieldsValue) || - isTypeObject(fieldsValue) - ); +const hasEarlyReturnConditions = (fieldsValue: FieldsType) => { + return fieldsValue.length === 0 || isNestedObject(fieldsValue) || isTypeObject(fieldsValue); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts index 68d0c2f047727..670fba02b2aec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts @@ -20,7 +20,9 @@ export type FieldsType = string[] | number[] | boolean[] | object[]; export type MergeStrategyFunction = ({ doc, ignoreFields, + ignoreFieldsRegexes, }: { doc: SignalSourceHit; - ignoreFields: string[]; + ignoreFields: Record; + ignoreFieldsRegexes: string[]; }) => SignalSourceHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.test.ts deleted file mode 100644 index 1495fd4b9aaca..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { buildFieldsKeyAsArrayMap } from './build_fields_key_as_array_map'; - -describe('buildFieldsKeyAsArrayMap()', () => { - it('returns primitive type if it passed as source', () => { - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(1)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(Infinity)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(NaN)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(false)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(null)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(undefined)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap([])).toEqual({}); - }); - it('builds map for nested source', () => { - expect(buildFieldsKeyAsArrayMap({ a: 'b' })).toEqual({ a: ['a'] }); - expect(buildFieldsKeyAsArrayMap({ a: ['b'] })).toEqual({ a: ['a'] }); - expect(buildFieldsKeyAsArrayMap({ a: { b: { c: 1 } } })).toEqual({ - a: ['a'], - 'a.b': ['a', 'b'], - 'a.b.c': ['a', 'b', 'c'], - }); - expect(buildFieldsKeyAsArrayMap({ a: { b: 'c' }, d: { e: 'f' } })).toEqual({ - a: ['a'], - 'a.b': ['a', 'b'], - d: ['d'], - 'd.e': ['d', 'e'], - }); - }); - - it('builds map for flattened source', () => { - expect(buildFieldsKeyAsArrayMap({ a: 'b' })).toEqual({ a: ['a'] }); - expect(buildFieldsKeyAsArrayMap({ 'a.b.c': 1 })).toEqual({ 'a.b.c': ['a.b.c'] }); - expect(buildFieldsKeyAsArrayMap({ 'a.b': 'c', 'd.e': 'f' })).toEqual({ - 'a.b': ['a.b'], - 'd.e': ['d.e'], - }); - }); - - it('builds map for arrays in a path', () => { - expect(buildFieldsKeyAsArrayMap({ a: { b: [{ c: 1 }, { c: 2 }] } })).toEqual({ - a: ['a'], - 'a.b': ['a', 'b'], - 'a.b.c': ['a', 'b', 'c'], - }); - }); - - it('builds map for mixed nested and flattened in a path', () => { - expect( - buildFieldsKeyAsArrayMap({ - 'a.b': { c: { d: 1 } }, - }) - ).toEqual({ - 'a.b': ['a.b'], - 'a.b.c': ['a.b', 'c'], - 'a.b.c.d': ['a.b', 'c', 'd'], - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.ts deleted file mode 100644 index 33e3814104921..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isPlainObject, isArray } from 'lodash'; - -import type { SearchTypes } from '../../../../../../../common/detection_engine/types'; - -const isObjectTypeGuard = (value: SearchTypes): value is Record => { - return isPlainObject(value); -}; - -function traverseSource( - document: SearchTypes, - result: Record = {}, - prefix: string[] = [] -): Record { - if (prefix.length) { - result[prefix.join('.')] = prefix; - } - - if (isObjectTypeGuard(document)) { - for (const [key, value] of Object.entries(document)) { - const path = [...prefix, key]; - - traverseSource(value, result, path); - } - } else if (isArray(document)) { - // for array of primitive values we can call traverseSource once - if (isPlainObject(document[0])) { - traverseSource(document[0], result, prefix); - } else { - document.forEach((doc) => { - traverseSource(doc, result, prefix); - }); - } - } - - return result; -} - -/** - * takes object document and creates map of string field keys to array field keys - * source `{ 'a.b': { c: { d: 1 } } }` - * will result in map: `{ - * 'a.b': ['a.b'], - * 'a.b.c': ['a.b', 'c'], - * 'a.b.c.d': ['a.b', 'c', 'd'], - * }` - * @param document - Record - **/ -export function buildFieldsKeyAsArrayMap( - document: Record -): Record { - if (!isPlainObject(document)) { - return {}; - } - - return traverseSource(document); -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.test.ts deleted file mode 100644 index 7288e7d1e9b80..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { filterFieldEntries } from './filter_field_entries'; -import type { FieldsType } from '../types'; - -describe('filter_field_entries', () => { - beforeAll(() => { - jest.resetAllMocks(); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - /** Dummy test value */ - const dummyValue = ['value']; - - /** - * Get the return type of the mergeFieldsWithSource for TypeScript checks against expected - */ - type ReturnTypeFilterFieldEntries = ReturnType; - - test('returns a single valid fieldEntries as expected', () => { - const fieldEntries: Array<[string, FieldsType]> = [['foo.bar', dummyValue]]; - expect(filterFieldEntries(fieldEntries, [])).toEqual( - fieldEntries - ); - }); - - test('removes invalid dotted entries', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['.', dummyValue], - ['foo.bar', dummyValue], - ['..', dummyValue], - ['foo..bar', dummyValue], - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo.bar', dummyValue], - ]); - }); - - test('removes multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['foo', dummyValue], - ['foo.keyword', dummyValue], // <-- "foo.keyword" multi-field should be removed - ['bar.keyword', dummyValue], // <-- "bar.keyword" multi-field should be removed - ['bar', dummyValue], - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo', dummyValue], - ['bar', dummyValue], - ]); - }); - - test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['host.name', dummyValue], - ['host.name.keyword', dummyValue], // <-- multi-field should be removed - ['host.hostname', dummyValue], - ['host.hostname.keyword', dummyValue], // <-- multi-field should be removed - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['host.name', dummyValue], - ['host.hostname', dummyValue], - ]); - }); - - test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['foo.host.name', dummyValue], - ['foo.host.name.keyword', dummyValue], // <-- multi-field should be removed - ['foo.host.hostname', dummyValue], - ['foo.host.hostname.keyword', dummyValue], // <-- multi-field should be removed - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo.host.name', dummyValue], - ['foo.host.hostname', dummyValue], - ]); - }); - - test('ignores fields of "_ignore", for eql bug https://github.com/elastic/elasticsearch/issues/77152', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['_ignored', dummyValue], - ['foo.host.hostname', dummyValue], - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo.host.hostname', dummyValue], - ]); - }); - - test('ignores fields given strings and regular expressions in the ignoreFields list', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['host.name', dummyValue], - ['user.name', dummyValue], // <-- string from ignoreFields should ignore this - ['host.hostname', dummyValue], - ['_odd.value', dummyValue], // <-- regular expression from ignoreFields should ignore this - ]; - expect( - filterFieldEntries(fieldEntries, ['user.name', '/[_]+/']) - ).toEqual([ - ['host.name', dummyValue], - ['host.hostname', dummyValue], - ]); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.test.ts new file mode 100644 index 0000000000000..b9ae033f425fa --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filterFieldEntry } from './filter_field_entry'; + +describe('filter_field_entry', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + /** Dummy test value */ + const dummyValue = ['value']; + + test('returns true for a valid field entry', () => { + const fieldsKeys: string[] = ['foo.bar']; + expect(filterFieldEntry(['foo.bar', dummyValue], fieldsKeys, {}, [])).toEqual(true); + }); + + test('returns false for invalid dotted entries', () => { + const fieldsKeys: string[] = ['.', 'foo.bar', '..', 'foo..bar']; + expect(filterFieldEntry(['.', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['foo.bar', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['..', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['foo..bar', dummyValue], fieldsKeys, {}, [])).toEqual(false); + }); + + test('removes multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { + const fieldsKeys: string[] = [ + 'foo', + 'foo.keyword', // <-- "foo.keyword" multi-field should be removed + 'bar.keyword', // <-- "bar.keyword" multi-field should be removed + 'bar', + ]; + expect(filterFieldEntry(['foo', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['foo.keyword', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['bar.keyword', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['bar', dummyValue], fieldsKeys, {}, [])).toEqual(true); + }); + + test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const fieldsKeys: string[] = [ + 'host.name', + 'host.name.keyword', // <-- multi-field should be removed + 'host.hostname', + 'host.hostname.keyword', // <-- multi-field should be removed + ]; + expect(filterFieldEntry(['host.name', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['host.name.keyword', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['host.hostname', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['host.hostname.keyword', dummyValue], fieldsKeys, {}, [])).toEqual( + false + ); + }); + + test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const fieldsKeys: string[] = [ + 'foo.host.name', + 'foo.host.name.keyword', // <-- multi-field should be removed + 'foo.host.hostname', + 'foo.host.hostname.keyword', // <-- multi-field should be removed + ]; + expect(filterFieldEntry(['foo.host.name', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['foo.host.name.keyword', dummyValue], fieldsKeys, {}, [])).toEqual( + false + ); + expect(filterFieldEntry(['foo.host.hostname', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['foo.host.hostname.keyword', dummyValue], fieldsKeys, {}, [])).toEqual( + false + ); + }); + + test('ignores fields of "_ignore", for eql bug https://github.com/elastic/elasticsearch/issues/77152', () => { + const fieldsKeys: string[] = ['_ignored', 'foo.host.hostname']; + expect(filterFieldEntry(['_ignored', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['foo.host.hostname', dummyValue], fieldsKeys, {}, [])).toEqual(true); + }); + + test('ignores fields given strings and regular expressions in the ignoreFields list', () => { + const fieldsKeys: string[] = [ + 'host.name', + 'user.name', // <-- string from ignoreFields should ignore this + 'host.hostname', + '_odd.value', // <-- regular expression from ignoreFields should ignore this + ]; + expect( + filterFieldEntry(['host.name', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(true); + expect( + filterFieldEntry(['user.name', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(false); + expect( + filterFieldEntry(['host.hostname', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(true); + expect( + filterFieldEntry(['_odd.value', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.ts similarity index 58% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.ts index 581dd7ffbff1a..3a18d36334070 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.ts @@ -18,23 +18,25 @@ import { isEqlBug77152 } from './is_eql_bug_77152'; * are invalid runtime field names. Also matches type objects such as geo-points and we ignore * those and don't try to merge those. * - * @param fieldEntries The field entries to filter + * @param fieldEntry The specific entry to test + * @param fieldEntries The full list of field entries, so we can check for multifields * @param ignoreFields Array of fields to ignore. If a value starts and ends with "/", such as: "/[_]+/" then the field will be treated as a regular expression. * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" - * @returns The field entries filtered + * @returns boolean Whether or not the field entry is valid for merging into _source */ -export const filterFieldEntries = ( - fieldEntries: Array<[string, FieldsType]>, - ignoreFields: string[] -): Array<[string, FieldsType]> => { - return fieldEntries.filter(([fieldsKey, fieldsValue]: [string, FieldsType]) => { - return ( - // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields - !isEqlBug77152(fieldsKey) && - !isIgnored(fieldsKey, ignoreFields) && - !isInvalidKey(fieldsKey) && - !isMultiField(fieldsKey, fieldEntries) && - !isTypeObject(fieldsValue) - ); - }); +export const filterFieldEntry = ( + fieldEntry: [string, FieldsType], + fieldsKeys: string[], + ignoreFields: Record, + ignoreFieldRegexes: string[] +): boolean => { + const [fieldsKey, fieldsValue] = fieldEntry; + // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields + return ( + !isEqlBug77152(fieldsKey) && + !isIgnored(fieldsKey, ignoreFields, ignoreFieldRegexes) && + !isInvalidKey(fieldsKey) && + !isMultiField(fieldsKey, fieldsKeys) && + !isTypeObject(fieldsValue) + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts index 87b1097dd9bca..4c48b585167b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts @@ -5,7 +5,7 @@ * 2.0. */ export * from './array_in_path_exists'; -export * from './filter_field_entries'; +export * from './filter_field_entry'; export * from './is_array_of_primitives'; export * from './is_ignored'; export * from './is_invalid_key'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts index e4a7093ef127c..a4aa55cbabcb1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts @@ -17,68 +17,72 @@ describe('is_ignored', () => { }); describe('string matching', () => { - test('it returns false if given an empty array', () => { - expect(isIgnored('simple.value', [])).toEqual(false); + test('it returns false if given an empty object and empty array', () => { + expect(isIgnored('simple.value', {}, [])).toEqual(false); }); test('it returns true if a simple string value matches', () => { - expect(isIgnored('simple.value', ['simple.value'])).toEqual(true); + expect(isIgnored('simple.value', { 'simple.value': true }, [])).toEqual(true); }); test('it returns false if a simple string value does not match', () => { - expect(isIgnored('simple', ['simple.value'])).toEqual(false); + expect(isIgnored('simple', { 'simple.value': true }, [])).toEqual(false); }); test('it returns true if a simple string value matches with two strings', () => { - expect(isIgnored('simple.value', ['simple.value', 'simple.second.value'])).toEqual(true); + expect( + isIgnored('simple.value', { 'simple.value': true, 'simple.second.value': true }, []) + ).toEqual(true); }); test('it returns true if a simple string value matches the second string', () => { - expect(isIgnored('simple.second.value', ['simple.value', 'simple.second.value'])).toEqual( - true - ); + expect( + isIgnored('simple.second.value', { 'simple.value': true, 'simple.second.value': true }, []) + ).toEqual(true); }); test('it returns false if a simple string value does not match two strings', () => { - expect(isIgnored('simple', ['simple.value', 'simple.second.value'])).toEqual(false); + expect( + isIgnored('simple', { 'simple.value': true, 'simple.second.value': true }, []) + ).toEqual(false); }); test('it returns true if mixed with a regular expression in the list', () => { - expect(isIgnored('simple', ['simple', '/[_]+/'])).toEqual(true); + expect(isIgnored('simple', { simple: true }, ['/[_]+/'])).toEqual(true); }); }); describe('regular expression matching', () => { test('it returns true if a simple regular expression matches', () => { - expect(isIgnored('_ignored', ['/[_]+/'])).toEqual(true); + expect(isIgnored('_ignored', {}, ['/[_]+/'])).toEqual(true); }); test('it returns false if a simple regular expression does not match', () => { - expect(isIgnored('simple', ['/[_]+/'])).toEqual(false); + expect(isIgnored('simple', {}, ['/[_]+/'])).toEqual(false); }); test('it returns true if a simple regular expression matches a longer string', () => { - expect(isIgnored('___ignored', ['/[_]+/'])).toEqual(true); + expect(isIgnored('___ignored', {}, ['/[_]+/'])).toEqual(true); }); test('it returns true if mixed with regular stings', () => { - expect(isIgnored('___ignored', ['simple', '/[_]+/'])).toEqual(true); + expect(isIgnored('___ignored', { simple: true }, ['/[_]+/'])).toEqual(true); }); test('it returns true with start anchor', () => { - expect(isIgnored('_ignored', ['simple', '/^[_]+/'])).toEqual(true); + expect(isIgnored('_ignored', { simple: true }, ['/^[_]+/'])).toEqual(true); }); test('it returns false with start anchor', () => { - expect(isIgnored('simple.something_', ['simple', '/^[_]+/'])).toEqual(false); + expect(isIgnored('simple.something_', { simple: true }, ['/^[_]+/'])).toEqual(false); }); test('it returns true with end anchor', () => { - expect(isIgnored('something_', ['simple', '/[_]+$/'])).toEqual(true); + expect(isIgnored('something_', { simple: true }, ['/[_]+$/'])).toEqual(true); }); test('it returns false with end anchor', () => { - expect(isIgnored('_something', ['simple', '/[_]+$/'])).toEqual(false); + expect(isIgnored('_something', { simple: true }, ['/[_]+$/'])).toEqual(false); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts index a418ce735626d..46638293bce17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts @@ -12,12 +12,13 @@ * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" * @returns true if it is a field to ignore, otherwise false */ -export const isIgnored = (fieldsKey: string, ignoreFields: string[]): boolean => { - return ignoreFields.some((ignoreField) => { - if (ignoreField.startsWith('/') && ignoreField.endsWith('/')) { - return new RegExp(ignoreField.slice(1, -1)).test(fieldsKey); - } else { - return fieldsKey === ignoreField; - } - }); +export const isIgnored = ( + fieldsKey: string, + ignoreFields: Record, + ignoreFieldRegexes: string[] +): boolean => { + return ( + (ignoreFields[fieldsKey] ? true : false) || + ignoreFieldRegexes.some((regex) => new RegExp(regex.slice(1, -1)).test(fieldsKey)) + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts index a8050b600b464..8b85557ee521a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts @@ -16,25 +16,23 @@ describe('is_multifield', () => { jest.resetAllMocks(); }); - const dummyValue = ['value']; - test('it returns true if the string "foo.bar" is a multiField', () => { - expect(isMultiField('foo.bar', [['foo', dummyValue]])).toEqual(true); + expect(isMultiField('foo.bar', ['foo'])).toEqual(true); }); test('it returns false if the string "foo" is not a multiField', () => { - expect(isMultiField('foo', [['foo', dummyValue]])).toEqual(false); + expect(isMultiField('foo', ['foo'])).toEqual(false); }); test('it returns false if we have a sibling string and are not a multiField', () => { - expect(isMultiField('foo.bar', [['foo.mar', dummyValue]])).toEqual(false); + expect(isMultiField('foo.bar', ['foo.mar'])).toEqual(false); }); test('it returns true for a 3rd level match of being a sub-object. Runtime fields can have multiple layers of multiFields', () => { - expect(isMultiField('foo.mars.bar', [['foo', dummyValue]])).toEqual(true); + expect(isMultiField('foo.mars.bar', ['foo'])).toEqual(true); }); test('it returns true for a 3rd level match against a 2nd level sub-object. Runtime fields can have multiple layers of multiFields', () => { - expect(isMultiField('foo.mars.bar', [['foo.mars', dummyValue]])).toEqual(true); + expect(isMultiField('foo.mars.bar', ['foo.mars'])).toEqual(true); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts index e9e51f9f50389..e19ff40a32e90 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts @@ -5,8 +5,6 @@ * 2.0. */ -import type { FieldsType } from '../types'; - /** * Returns true if we are a multiField when passed in a fields entry and a fields key, * otherwise false. Notice that runtime fields can have multiple levels of multiFields which is kind a problem @@ -16,18 +14,15 @@ import type { FieldsType } from '../types'; * @param fieldEntries The entries to check against. * @returns True if we are a subObject, otherwise false. */ -export const isMultiField = ( - fieldsKey: string, - fieldEntries: Array<[string, FieldsType]> -): boolean => { +export const isMultiField = (fieldsKey: string, fieldsKeys: string[]): boolean => { const splitPath = fieldsKey.split('.'); return splitPath.some((_, index, array) => { if (index + 1 === array.length) { return false; } else { const newPath = [...array].splice(0, index + 1).join('.'); - return fieldEntries.some(([fieldKeyToCheck]) => { - return fieldKeyToCheck === newPath; + return fieldsKeys.some((key) => { + return key === newPath; }); } }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts index 39d215d4c5062..be4f35f1948f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts @@ -6,6 +6,7 @@ */ import { isObjectLike } from 'lodash/fp'; +import { isPlainObject } from 'lodash'; import type { SearchTypes } from '../../../../../../../common/detection_engine/types'; /** @@ -23,3 +24,7 @@ export const isObjectLikeOrArrayOfObjectLikes = ( return isObjectLike(valueInMergedDocument); } }; + +export const isObjectTypeGuard = (value: SearchTypes): value is Record => { + return isPlainObject(value); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts index 896adb2326a8f..6cdac6c1662e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts @@ -5,138 +5,97 @@ * 2.0. */ -import { isPathValid } from './is_path_valid'; - -describe('isPathValid', () => { - test('not valid when empty array is key', () => { - expect(isPathValid([], {})).toEqual(false); - }); +import { robustIsPathValid } from './is_path_valid'; +describe('robustIsPathValid', () => { test('valid when empty string is key', () => { - expect(isPathValid('', {})).toEqual(true); - expect(isPathValid([''], {})).toEqual(true); + expect(robustIsPathValid('', {})).toEqual(true); }); test('valid when a path and empty object', () => { - expect(isPathValid(['a', 'b', 'c'], {})).toEqual(true); - expect(isPathValid('a.b.c', {})).toEqual(true); + expect(robustIsPathValid('a.b.c', {})).toEqual(true); }); test('not valid when a path and an array exists', () => { - expect(isPathValid(['a'], { a: [] })).toEqual(false); - expect(isPathValid('a', { a: [] })).toEqual(false); + expect(robustIsPathValid('a', { a: [] })).toEqual(false); }); test('not valid when a path and primitive value exists', () => { - expect(isPathValid(['a'], { a: 'test' })).toEqual(false); - expect(isPathValid(['a'], { a: 1 })).toEqual(false); - expect(isPathValid(['a'], { a: true })).toEqual(false); - - expect(isPathValid('a', { a: 'test' })).toEqual(false); - expect(isPathValid('a', { a: 1 })).toEqual(false); - expect(isPathValid('a', { a: true })).toEqual(false); + expect(robustIsPathValid('a', { a: 'test' })).toEqual(false); + expect(robustIsPathValid('a', { a: 1 })).toEqual(false); + expect(robustIsPathValid('a', { a: true })).toEqual(false); }); test('valid when a path and object value exists', () => { - expect(isPathValid(['a'], { a: {} })).toEqual(true); - - expect(isPathValid('a', { a: {} })).toEqual(true); + expect(robustIsPathValid('a', { a: {} })).toEqual(true); }); test('not valid when a path and an array exists within the parent path at level 1', () => { - expect(isPathValid(['a', 'b'], { a: [] })).toEqual(false); - - expect(isPathValid('a.b', { a: [] })).toEqual(false); + expect(robustIsPathValid('a.b', { a: [] })).toEqual(false); }); test('not valid when a path and primitive value exists within the parent path at level 1', () => { - expect(isPathValid(['a', 'b'], { a: 'test' })).toEqual(false); - expect(isPathValid(['a', 'b'], { a: 1 })).toEqual(false); - expect(isPathValid(['a', 'b'], { a: true })).toEqual(false); - - expect(isPathValid('a.b', { a: 'test' })).toEqual(false); - expect(isPathValid('a.b', { a: 1 })).toEqual(false); - expect(isPathValid('a.b', { a: true })).toEqual(false); + expect(robustIsPathValid('a.b', { a: 'test' })).toEqual(false); + expect(robustIsPathValid('a.b', { a: 1 })).toEqual(false); + expect(robustIsPathValid('a.b', { a: true })).toEqual(false); }); test('valid when a path and object value exists within the parent path at level 1', () => { - expect(isPathValid(['a', 'b'], { a: {} })).toEqual(true); - - expect(isPathValid('a.b', { a: {} })).toEqual(true); + expect(robustIsPathValid('a.b', { a: {} })).toEqual(true); }); test('not valid when a path and an array exists within the parent path at level 2', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: [] } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { 'a.b': [] })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: [] } })).toEqual(false); - expect(isPathValid('a.b.c', { 'a.b': [] })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': [] })).toEqual(false); }); test('not valid when a path and primitive value exists within the parent path at level 2', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: 'test' } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: 1 } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: true } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { 'a.b': true })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: 'test' } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: 1 } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: true } })).toEqual(false); - expect(isPathValid('a.b.c', { 'a.b': true })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: 'test' } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: 1 } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: true } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': true })).toEqual(false); }); test('valid when a path and object value exists within the parent path at the last level 2', () => { - expect(isPathValid(['a', 'b'], { a: { b: {} } })).toEqual(true); - - expect(isPathValid('a.b', { a: { b: {} } })).toEqual(true); + expect(robustIsPathValid('a.b', { a: { b: {} } })).toEqual(true); }); test('not valid when a path and an array exists within the parent path at the last level 3', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: [] } } })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: { c: [] } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: [] } } })).toEqual(false); }); test('not valid when a path and primitive value exists within the parent path at the last level 3', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: 'test' } } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: 1 } } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: true } } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { 'a.b.c': true })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: { c: 'test' } } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: { c: 1 } } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: { c: true } } })).toEqual(false); - expect(isPathValid('a.b.c', { 'a.b.c': true })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: 'test' } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: 1 } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: true } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b.c': true })).toEqual(false); }); test('valid when a path and object value exists within the parent path at the last level 3', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: {} } } })).toEqual(true); - expect(isPathValid(['a', 'b', 'c'], { 'a.b.c': {} })).toEqual(true); - - expect(isPathValid('a.b.c', { a: { b: { c: {} } } })).toEqual(true); - expect(isPathValid('a.b.c', { 'a.b.c': {} })).toEqual(true); + expect(robustIsPathValid('a.b.c', { a: { b: { c: {} } } })).toEqual(true); + expect(robustIsPathValid('a.b.c', { 'a.b.c': {} })).toEqual(true); }); - test('valid when any key has dot notation', () => { - expect(isPathValid(['a', 'b.c'], { a: { 'b.c': {} } })).toEqual(true); - expect(isPathValid(['a.b', 'c'], { 'a.b': { c: {} } })).toEqual(true); - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': { d: {} } } })).toEqual(true); + test('valid when using dot notation', () => { + expect(robustIsPathValid('a.b.c', { a: { 'b.c': {} } })).toEqual(true); + expect(robustIsPathValid('a.b.c', { 'a.b': { c: {} } })).toEqual(true); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': { d: {} } } })).toEqual(true); }); - test('not valid when any key has dot notation and array is present in source on the last level', () => { - expect(isPathValid(['a', 'b.c'], { a: { 'b.c': [] } })).toEqual(false); - expect(isPathValid(['a.b', 'c'], { 'a.b': { c: [] } })).toEqual(false); - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': { d: [] } } })).toEqual(false); + test('not valid when using dot notation and array is present in source on the last level', () => { + expect(robustIsPathValid('a.b.c', { a: { 'b.c': [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': { c: [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': { d: [] } } })).toEqual(false); }); test('not valid when any key has dot notation and primitive value is present in source on the last level', () => { - expect(isPathValid(['a', 'b.c'], { a: { 'b.c': 1 } })).toEqual(false); - expect(isPathValid(['a.b', 'c'], { 'a.b': { c: 1 } })).toEqual(false); - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': { d: 1 } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { 'b.c': 1 } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': { c: 1 } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': { d: 1 } } })).toEqual(false); }); test('not valid when any key has dot notation and array is present in source on level 2', () => { - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': [] } })).toEqual(false); - expect(isPathValid(['a.b', 'c', 'd'], { 'a.b': { c: [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { 'a.b': { c: [] } })).toEqual(false); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts index 6e193e237696d..f9c87088bd25b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts @@ -5,32 +5,32 @@ * 2.0. */ -import { get, isPlainObject } from 'lodash/fp'; -import type { SignalSource } from '../../../types'; +import { isObjectTypeGuard } from './is_objectlike_or_array_of_objectlikes'; /** * Returns true if path in SignalSource object is valid * Path is valid if each field in hierarchy is object or undefined * Path is not valid if ANY of field in hierarchy is not object or undefined - * @param path in source to check within source - * @param source The source document + * The function is robust in that it can handle any mix of dot and nested notation in the document + * @param key Path (dot-notation) to check for validity + * @param document Document to search * @returns boolean */ -export const isPathValid = (path: string[] | string, source: SignalSource): boolean => { - if (path == null) { - return false; +export const robustIsPathValid = (key: string, document: Record): boolean => { + const splitKey = key.split('.'); + let tempKey = splitKey[0]; + for (let i = 0; i < splitKey.length; i++) { + if (i > 0) { + tempKey += `.${splitKey[i]}`; + } + const value = document[tempKey]; + if (value != null) { + if (!isObjectTypeGuard(value)) { + return false; + } else { + return robustIsPathValid(splitKey.slice(i + 1).join('.'), value); + } + } } - const pathAsArray = typeof path === 'string' ? path.split('.') : path; - - if (pathAsArray.length === 0) { - return false; - } - - return pathAsArray.every((_, index, array) => { - const newPath = [...array].splice(0, index + 1); - // _.get won't retrieve value of flattened key 'a.b' when receives path ['a', 'b']. - // so we would try to call _.get with dot-notation path if array path results in undefined - const valueToCheck = get(newPath, source) ?? get(newPath.join('.'), source); - return valueToCheck === undefined || isPlainObject(valueToCheck); - }); + return true; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts new file mode 100644 index 0000000000000..646068b59eec7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { robustGet, robustSet } from './robust_field_access'; + +describe('robust field access', () => { + describe('get', () => { + it('fetches a value with basic key', () => { + expect(robustGet({ key: 'a.b.c', document: { a: { b: { c: 'my-value' } } } })).toEqual( + 'my-value' + ); + }); + + it('fetches a value with mixed notation', () => { + expect(robustGet({ key: 'a.b.c', document: { 'a.b': { c: 'my-value' } } })).toEqual( + 'my-value' + ); + }); + + it('fetches a value with different mixed notation', () => { + expect(robustGet({ key: 'a.b.c', document: { a: { 'b.c': 'my-value' } } })).toEqual( + 'my-value' + ); + }); + + it('fetches a value using only dot notation', () => { + expect(robustGet({ key: 'a.b.c', document: { 'a.b.c': 'my-value' } })).toEqual('my-value'); + }); + + it('returns undefined if the key does not exist', () => { + expect(robustGet({ key: 'a.b.c', document: { a: { b: 'my-value' } } })).toEqual(undefined); + }); + }); + + describe('set', () => { + it('sets a value with a basic key', () => { + expect(robustSet({ key: 'a.b.c', valueToSet: 'test-value', document: {} })).toEqual({ + a: { b: { c: 'test-value' } }, + }); + }); + + it('sets a value inside an object at a dot notation path', () => { + expect( + robustSet({ key: 'a.b.c', valueToSet: 'test-value', document: { 'a.b': {} } }) + ).toEqual({ + 'a.b': { c: 'test-value' }, + }); + }); + + it('sets a value inside an object at a nested notation path', () => { + expect( + robustSet({ key: 'a.b.c', valueToSet: 'test-value', document: { a: { b: {} } } }) + ).toEqual({ + a: { b: { c: 'test-value' } }, + }); + }); + + it('sets a value and overwrites the existing value with dot notation', () => { + expect( + robustSet({ key: 'a.b.c', valueToSet: 'test-new', document: { 'a.b.c': 'test-original' } }) + ).toEqual({ + 'a.b.c': 'test-new', + }); + }); + + it('sets a value and overwrites the existing value with nested notation', () => { + expect( + robustSet({ + key: 'a.b.c', + valueToSet: 'test-new', + document: { a: { b: { c: 'test-original' } } }, + }) + ).toEqual({ + a: { b: { c: 'test-new' } }, + }); + }); + + it('sets a value and overwrites the existing value with mixed notation', () => { + expect( + robustSet({ + key: 'a.b.c', + valueToSet: 'test-new', + document: { 'a.b': { c: 'test-original' } }, + }) + ).toEqual({ + 'a.b': { c: 'test-new' }, + }); + }); + + it('sets a value and ignores non-object values on the path', () => { + expect( + robustSet({ + key: 'a.b.c', + valueToSet: 'test-new', + document: { 'a.b': 'test-ignore' }, + }) + ).toEqual({ + 'a.b': 'test-ignore', + a: { b: { c: 'test-new' } }, + }); + }); + + it('sets the value correctly if an object already exists at the path', () => { + expect( + robustSet({ + key: 'a.b', + valueToSet: 'test-new', + document: { 'a.b': {} }, + }) + ).toEqual({ 'a.b': 'test-new' }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts new file mode 100644 index 0000000000000..30d716896056d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@kbn/safer-lodash-set'; + +import type { SearchTypes } from '../../../../../../../common/detection_engine/types'; +import { isObjectTypeGuard } from './is_objectlike_or_array_of_objectlikes'; + +/** + * Similar to lodash `get`, but instead of handling only pure dot or nested notation this function handles any mix of dot and nested notation + * + * Note: this function makes no attempt to handle arrays in the middle of the path because it's only used to fetch values based on paths from + * the `fields` option on search requests, which can never have arrays in the middle of the path + * @param key Path to field, in dot notation + * @param document Object to fetch value from + * @returns Fetched value or undefined + */ +export const robustGet = ({ + key, + document, +}: { + key: string; + document: Record; +}): SearchTypes => { + const fastPathValue = document[key]; + if (fastPathValue != null) { + return fastPathValue; + } + const splitKey = key.split('.'); + let tempKey = splitKey[0]; + for (let i = 0; i < splitKey.length - 1; i++) { + if (i > 0) { + tempKey += `.${splitKey[i]}`; + } + const value = document[tempKey]; + if (value != null) { + if (isObjectTypeGuard(value)) { + return robustGet({ key: splitKey.slice(i + 1).join('.'), document: value }); + } else { + return undefined; + } + } + } + return undefined; +}; + +/** + * Similar to lodash set, but instead of handling only pure dot or nested notation this function handles any mix of dot and nested notation + * @param key Path to field, in dot notation + * @param valueToSet Value to insert into document + * @param document Object to insert value into + * @returns Updated document + */ +export const robustSet = >({ + key, + valueToSet, + document, +}: { + key: string; + valueToSet: SearchTypes; + document: T; +}) => { + const splitKey = key.split('.'); + let tempKey = splitKey[0]; + for (let i = 0; i < splitKey.length - 1; i++) { + if (i > 0) { + tempKey += `.${splitKey[i]}`; + } + const value = document[tempKey]; + if (value != null) { + if (isObjectTypeGuard(value)) { + robustSet({ key: splitKey.slice(i + 1).join('.'), valueToSet, document: value }); + return document; + } + } + } + return set(document, key, valueToSet); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts index 5baa84ff913b2..4d47c279f1495 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts @@ -16,13 +16,13 @@ import { } from '@kbn/rule-data-utils'; import type { CompleteRule, ThreatRuleParams } from '../../rule_schema'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; -jest.mock('../factories/utils/build_bulk_body', () => ({ buildBulkBody: jest.fn() })); +jest.mock('../factories/utils/transform_hit_to_alert', () => ({ transformHitToAlert: jest.fn() })); -const buildBulkBodyMock = buildBulkBody as jest.Mock; +const transformHitToAlertMock = transformHitToAlert as jest.Mock; const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); @@ -118,7 +118,7 @@ const wrappedParams = { }; describe('wrapSuppressedAlerts', () => { - buildBulkBodyMock.mockReturnValue({ 'mock-props': true }); + transformHitToAlertMock.mockReturnValue({ 'mock-props': true }); it('should wrap event with alert fields and correctly set suppression fields', () => { const expectedTimestamp = '2020-10-28T06:30:00.000Z'; @@ -137,10 +137,10 @@ describe('wrapSuppressedAlerts', () => { ...wrappedParams, }); - expect(buildBulkBodyMock).toHaveBeenCalledWith( - 'default', - wrappedParams.completeRule, - { + expect(transformHitToAlertMock).toHaveBeenCalledWith({ + spaceId: 'default', + completeRule: wrappedParams.completeRule, + doc: { fields: { '@timestamp': [expectedTimestamp], 'agent.name': ['agent-0'], @@ -149,16 +149,17 @@ describe('wrapSuppressedAlerts', () => { _id: '1', _index: 'test*', }, - 'missingFields', - [], - true, - wrappedParams.buildReasonMessage, - ['test*'], - undefined, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: wrappedParams.buildReasonMessage, + indicesToQuery: ['test*'], + alertTimestampOverride: undefined, ruleExecutionLogger, - expect.any(String), - 'public-url-mock' - ); + alertUuid: expect.any(String), + publicBaseUrl: 'public-url-mock', + }); expect(wrappedAlerts[0]._source).toEqual( expect.objectContaining({ [ALERT_SUPPRESSION_TERMS]: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 70fee20116fc4..87d1e64d8ece9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -23,7 +23,7 @@ import type { ThreatRuleParams, } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; import { generateId } from './utils'; @@ -77,20 +77,21 @@ export const wrapSuppressedAlerts = ({ const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/spaces/public/config.ts b/x-pack/plugins/spaces/public/config.ts index dcd203eb696e3..3dd2d3bc89f92 100644 --- a/x-pack/plugins/spaces/public/config.ts +++ b/x-pack/plugins/spaces/public/config.ts @@ -9,4 +9,7 @@ export interface ConfigType { maxSpaces: number; allowFeatureVisibility: boolean; allowSolutionVisibility: boolean; + experimental: { + forceSolutionVisibility: boolean; + }; } diff --git a/x-pack/plugins/spaces/public/management/management_service.test.ts b/x-pack/plugins/spaces/public/management/management_service.test.ts index 5f97515171518..e5438c0cf5e9c 100644 --- a/x-pack/plugins/spaces/public/management/management_service.test.ts +++ b/x-pack/plugins/spaces/public/management/management_service.test.ts @@ -24,6 +24,9 @@ describe('ManagementService', () => { maxSpaces: 1000, allowFeatureVisibility: true, allowSolutionVisibility: true, + experimental: { + forceSolutionVisibility: false, + }, }; describe('#setup', () => { diff --git a/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx b/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx index 61619c499181c..d9852a82f8259 100644 --- a/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx +++ b/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx @@ -31,6 +31,9 @@ const config: ConfigType = { maxSpaces: 1000, allowFeatureVisibility: true, allowSolutionVisibility: true, + experimental: { + forceSolutionVisibility: false, + }, }; const eventTracker = new EventTracker({ reportEvent: jest.fn() }); diff --git a/x-pack/plugins/spaces/public/mocks.ts b/x-pack/plugins/spaces/public/mocks.ts index 1499384f4ed72..8478ed010cd98 100644 --- a/x-pack/plugins/spaces/public/mocks.ts +++ b/x-pack/plugins/spaces/public/mocks.ts @@ -16,6 +16,7 @@ const createApiMock = (hasOnlyDefaultSpace: boolean): jest.Mocked => getActiveSpace: jest.fn(), ui: createApiUiMock(), hasOnlyDefaultSpace, + isSolutionViewEnabled: true, }); type SpacesApiUiMock = Omit, 'components'> & { diff --git a/x-pack/plugins/spaces/public/plugin.test.ts b/x-pack/plugins/spaces/public/plugin.test.ts index 9afe7d27e9a04..33565748a99e3 100644 --- a/x-pack/plugins/spaces/public/plugin.test.ts +++ b/x-pack/plugins/spaces/public/plugin.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { cloudMock } from '@kbn/cloud-plugin/public/mocks'; import { coreMock } from '@kbn/core/public/mocks'; import { homePluginMock } from '@kbn/home-plugin/public/mocks'; import { @@ -13,7 +14,6 @@ import { } from '@kbn/management-plugin/public/mocks'; import { SpacesPlugin } from './plugin'; -// import { ConfigSchema } from './config'; describe('Spaces plugin', () => { describe('#setup', () => { @@ -209,27 +209,156 @@ describe('Spaces plugin', () => { }); }); - it('determines hasOnlyDefaultSpace correctly when maxSpaces=1', () => { - const coreSetup = coreMock.createSetup(); - const coreStart = coreMock.createStart(); + describe('hasOnlyDefaultSpace', () => { + it('determines hasOnlyDefaultSpace correctly when maxSpaces=1', () => { + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); - const plugin = new SpacesPlugin(coreMock.createPluginInitializerContext({ maxSpaces: 1 })); - const spacesSetup = plugin.setup(coreSetup, {}); - const spacesStart = plugin.start(coreStart); + const plugin = new SpacesPlugin(coreMock.createPluginInitializerContext({ maxSpaces: 1 })); + const spacesSetup = plugin.setup(coreSetup, {}); + const spacesStart = plugin.start(coreStart); - expect(spacesSetup.hasOnlyDefaultSpace).toBe(true); - expect(spacesStart.hasOnlyDefaultSpace).toBe(true); + expect(spacesSetup.hasOnlyDefaultSpace).toBe(true); + expect(spacesStart.hasOnlyDefaultSpace).toBe(true); + }); + + it('determines hasOnlyDefaultSpace correctly when maxSpaces=1000', () => { + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); + + const plugin = new SpacesPlugin(coreMock.createPluginInitializerContext({ maxSpaces: 1000 })); + const spacesSetup = plugin.setup(coreSetup, {}); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.hasOnlyDefaultSpace).toBe(false); + expect(spacesStart.hasOnlyDefaultSpace).toBe(false); + }); }); - it('determines hasOnlyDefaultSpace correctly when maxSpaces=1000', () => { - const coreSetup = coreMock.createSetup(); - const coreStart = coreMock.createStart(); + describe('isSolutionViewEnabled', () => { + it('when onCloud, not serverless and allowSolutionVisibility is "true"', () => { + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); + const cloud = cloudMock.createSetup(); + cloud.isCloudEnabled = true; + + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: true }, + { buildFlavor: 'traditional' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, { cloud }); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(true); + expect(spacesStart.isSolutionViewEnabled).toBe(true); + }); - const plugin = new SpacesPlugin(coreMock.createPluginInitializerContext({ maxSpaces: 1000 })); - const spacesSetup = plugin.setup(coreSetup, {}); - const spacesStart = plugin.start(coreStart); + it('when not onCloud and allowSolutionVisibility is "true"', () => { + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); - expect(spacesSetup.hasOnlyDefaultSpace).toBe(false); - expect(spacesStart.hasOnlyDefaultSpace).toBe(false); + { + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: true }, // it is true but we are not onCloud + { buildFlavor: 'traditional' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, {}); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(false); // so it should be false + expect(spacesStart.isSolutionViewEnabled).toBe(false); + } + + { + // unless the forceSolutionVisibility flag is set + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: false, experimental: { forceSolutionVisibility: true } }, + { buildFlavor: 'traditional' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, {}); // we are not onCloud but forceSolutionVisibility is true + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(true); + expect(spacesStart.isSolutionViewEnabled).toBe(true); + } + }); + + it('when onCloud, not serverless and allowSolutionVisibility is "false"', () => { + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); + const cloud = cloudMock.createSetup(); + cloud.isCloudEnabled = true; + + { + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: false }, + { buildFlavor: 'traditional' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, { cloud }); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(false); + expect(spacesStart.isSolutionViewEnabled).toBe(false); + } + + { + // unless the forceSolutionVisibility flag is set + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: false, experimental: { forceSolutionVisibility: true } }, + { buildFlavor: 'traditional' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, { cloud }); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(true); + expect(spacesStart.isSolutionViewEnabled).toBe(true); + } + }); + + it('when onCloud and serverless', () => { + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); + const cloud = cloudMock.createSetup(); + cloud.isCloudEnabled = true; + + { + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: true }, + { buildFlavor: 'serverless' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, { cloud }); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(false); + expect(spacesStart.isSolutionViewEnabled).toBe(false); + } + + { + // unless the forceSolutionVisibility flag is set + const plugin = new SpacesPlugin( + coreMock.createPluginInitializerContext( + { allowSolutionVisibility: true, experimental: { forceSolutionVisibility: true } }, + { buildFlavor: 'serverless' } + ) + ); + const spacesSetup = plugin.setup(coreSetup, { cloud }); + const spacesStart = plugin.start(coreStart); + + expect(spacesSetup.isSolutionViewEnabled).toBe(true); + expect(spacesStart.isSolutionViewEnabled).toBe(true); + } + }); }); }); diff --git a/x-pack/plugins/spaces/public/plugin.tsx b/x-pack/plugins/spaces/public/plugin.tsx index 31ba324b926f3..6d545cdb70e61 100644 --- a/x-pack/plugins/spaces/public/plugin.tsx +++ b/x-pack/plugins/spaces/public/plugin.tsx @@ -60,6 +60,14 @@ export class SpacesPlugin implements Plugin, plugins: PluginsSetup) { const hasOnlyDefaultSpace = this.config.maxSpaces === 1; + const onCloud = plugins.cloud !== undefined && plugins.cloud.isCloudEnabled; + + // We only allow "solution" to be set on cloud environments, not on prem + // unless the forceSolutionVisibility flag is set + const allowSolutionVisibility = + (onCloud && !this.isServerless && this.config.allowSolutionVisibility) || + Boolean(this.config.experimental?.forceSolutionVisibility); + this.spacesManager = new SpacesManager(core.http); this.spacesApi = { ui: getUiApi({ @@ -69,15 +77,14 @@ export class SpacesPlugin implements Plugin this.spacesManager.onActiveSpaceChange$, getActiveSpace: () => this.spacesManager.getActiveSpace(), hasOnlyDefaultSpace, + isSolutionViewEnabled: allowSolutionVisibility, + }; + + this.config = { + ...this.config, + allowSolutionVisibility, }; - const onCloud = plugins.cloud !== undefined && plugins.cloud.isCloudEnabled; - if (!onCloud) { - this.config = { - ...this.config, - allowSolutionVisibility: false, - }; - } registerSpacesEventTypes(core); this.eventTracker = new EventTracker(core.analytics); @@ -133,7 +140,7 @@ export class SpacesPlugin implements Plugin { "allowFeatureVisibility": true, "allowSolutionVisibility": true, "enabled": true, + "experimental": Object { + "forceSolutionVisibility": false, + }, "maxSpaces": 1000, } `); @@ -32,6 +35,9 @@ describe('config schema', () => { "allowFeatureVisibility": true, "allowSolutionVisibility": true, "enabled": true, + "experimental": Object { + "forceSolutionVisibility": false, + }, "maxSpaces": 1000, } `); @@ -41,6 +47,9 @@ describe('config schema', () => { "allowFeatureVisibility": true, "allowSolutionVisibility": true, "enabled": true, + "experimental": Object { + "forceSolutionVisibility": false, + }, "maxSpaces": 1000, } `); diff --git a/x-pack/plugins/spaces/server/config.ts b/x-pack/plugins/spaces/server/config.ts index a7c9606e74543..ef6b300d43965 100644 --- a/x-pack/plugins/spaces/server/config.ts +++ b/x-pack/plugins/spaces/server/config.ts @@ -54,6 +54,13 @@ export const ConfigSchema = schema.object({ defaultValue: true, }), }), + experimental: schema.maybe( + offeringBasedSchema({ + traditional: schema.object({ + forceSolutionVisibility: schema.boolean({ defaultValue: false }), + }), + }) + ), }); export function createConfig$(context: PluginInitializerContext) { diff --git a/x-pack/plugins/spaces/server/index.ts b/x-pack/plugins/spaces/server/index.ts index a568f52c7c29a..297a99a525ec7 100644 --- a/x-pack/plugins/spaces/server/index.ts +++ b/x-pack/plugins/spaces/server/index.ts @@ -34,6 +34,9 @@ export const config: PluginConfigDescriptor = { maxSpaces: true, allowFeatureVisibility: true, allowSolutionVisibility: true, + experimental: { + forceSolutionVisibility: true, + }, }, }; diff --git a/x-pack/plugins/spaces/server/plugin.ts b/x-pack/plugins/spaces/server/plugin.ts index 8b91dcac35917..2f8fb2ec30842 100644 --- a/x-pack/plugins/spaces/server/plugin.ts +++ b/x-pack/plugins/spaces/server/plugin.ts @@ -130,7 +130,10 @@ export class SpacesPlugin ([config, onCloud]): ConfigType => ({ ...config, // We only allow "solution" to be set on cloud environments, not on prem - allowSolutionVisibility: onCloud ? config.allowSolutionVisibility : false, + // unless the forceSolutionVisibility flag is set. + allowSolutionVisibility: + (onCloud && config.allowSolutionVisibility) || + Boolean(config.experimental?.forceSolutionVisibility), }) ) ); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 1c1ebd739af62..5a4660f362987 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -2406,11 +2406,6 @@ "discover.docExplorerCallout.headerMessage": "Une meilleure façon d'explorer", "discover.docExplorerCallout.learnMore": "En savoir plus", "discover.docExplorerCallout.tryDocumentExplorer": "Testez l'explorateur de documents", - "discover.docExplorerUpdateCallout.closeButtonAriaLabel": "Fermer", - "discover.docExplorerUpdateCallout.description": "Ajoutez des champs pertinents, réorganisez et triez les colonnes, redimensionnez les lignes, et bien plus encore dans le tableau de documents.", - "discover.docExplorerUpdateCallout.dismissButtonLabel": "Rejeter", - "discover.docExplorerUpdateCallout.takeTourButtonLabel": "Découvrir", - "discover.docExplorerUpdateCallout.title": "Optimiser l'affichage de vos résultats de recherche", "discover.docTable.documentsNavigation": "Navigation dans les documents", "discover.docTable.limitedSearchResultLabel": "Limité à {resultCount} résultats. Veuillez affiner votre recherche.", "discover.docTable.noResultsTitle": "Résultat introuvable", @@ -2440,21 +2435,6 @@ "discover.documentsAriaLabel": "Documents", "discover.docViews.table.scoreSortWarningTooltip": "Filtrez sur _score pour pouvoir récupérer les valeurs correspondantes.", "discover.dropZoneTableLabel": "Abandonner la zone pour ajouter un champ en tant que colonne dans la table", - "discover.dscTour.stepAddFields.description": "Cliquez sur {plusIcon} pour ajouter les champs qui vous intéressent.", - "discover.dscTour.stepAddFields.imageAltText": "Dans la liste Champs disponibles, cliquez sur l'icône Plus pour afficher/masquer un champ dans le tableau de documents.", - "discover.dscTour.stepAddFields.title": "Ajouter des champs dans le tableau", - "discover.dscTour.stepChangeRowHeight.description": "Ajustez le nombre de lignes pour adapter la taille au contenu.", - "discover.dscTour.stepChangeRowHeight.imageAltText": "Cliquez sur l'icône d'options d'affichage pour ajuster la hauteur de ligne afin d'adapter la taille au contenu.", - "discover.dscTour.stepChangeRowHeight.title": "Modifier la hauteur de ligne", - "discover.dscTour.stepExpand.expandIconAriaLabel": "icône de développement", - "discover.dscTour.stepExpand.imageAltText": "Cliquez sur l'icône de développement pour inspecter et filtrer les champs du document et afficher le document en contexte.", - "discover.dscTour.stepExpand.title": "Développer les documents", - "discover.dscTour.stepReorderColumns.description": "Faites glisser les colonnes dans l'ordre souhaité.", - "discover.dscTour.stepReorderColumns.imageAltText": "Utilisez la fenêtre contextuelle Colonnes pour faire glisser les colonnes vers l'ordre souhaité.", - "discover.dscTour.stepReorderColumns.title": "Classer les colonnes de tableau", - "discover.dscTour.stepSort.description": "Utilisez le titre de colonne pour effectuer le tri sur un champ unique, ou la fenêtre contextuelle pour plusieurs champs.", - "discover.dscTour.stepSort.imageAltText": "Cliquez sur un en-tête de colonne et sélectionnez l'ordre de tri souhaité. Ajustez un tri à champ multiple à l'aide de la fenêtre contextuelle des champs triés.", - "discover.dscTour.stepSort.title": "Trier sur un ou plusieurs champs", "discover.embeddable.inspectorRequestDataTitle": "Données", "discover.embeddable.inspectorRequestDescription": "Cette requête interroge Elasticsearch afin de récupérer les données pour la recherche.", "discover.embeddable.search.displayName": "rechercher", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index fbe9e93e8a768..c112b137c2e2b 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -2406,11 +2406,6 @@ "discover.docExplorerCallout.headerMessage": "より効率的な探索方法", "discover.docExplorerCallout.learnMore": "詳細", "discover.docExplorerCallout.tryDocumentExplorer": "ドキュメントエクスプローラーを試す", - "discover.docExplorerUpdateCallout.closeButtonAriaLabel": "閉じる", - "discover.docExplorerUpdateCallout.description": "ドキュメントテーブルで関連するフィールドの追加、列の並べ替え、行のサイズ変更などを行います。", - "discover.docExplorerUpdateCallout.dismissButtonLabel": "閉じる", - "discover.docExplorerUpdateCallout.takeTourButtonLabel": "ツアーを表示", - "discover.docExplorerUpdateCallout.title": "検索結果を最も効果的に表示", "discover.docTable.documentsNavigation": "ドキュメントナビゲーション", "discover.docTable.limitedSearchResultLabel": "{resultCount}件の結果のみが表示されます。検索結果を絞り込みます。", "discover.docTable.noResultsTitle": "結果が見つかりませんでした", @@ -2439,22 +2434,6 @@ "discover.documentsAriaLabel": "ドキュメント", "discover.docViews.table.scoreSortWarningTooltip": "_scoreの値を取得するには、並べ替える必要があります。", "discover.dropZoneTableLabel": "フィールドを列として表に追加するには、ゾーンをドロップします", - "discover.dscTour.stepAddFields.description": "{plusIcon}をクリックして、関心があるフィールドを追加します。", - "discover.dscTour.stepAddFields.imageAltText": "［使用可能なフィールド］リストで、プラスアイコンをクリックし、フィールドをドキュメントテーブルに切り替えます。", - "discover.dscTour.stepAddFields.title": "フィールドをテーブルに追加", - "discover.dscTour.stepChangeRowHeight.description": "コンテンツに合わせて行数を調整します。", - "discover.dscTour.stepChangeRowHeight.imageAltText": "表示オプションアイコンをクリックし、行高さを調整して、コンテンツに合わせます。", - "discover.dscTour.stepChangeRowHeight.title": "行高さの変更", - "discover.dscTour.stepExpand.description": "{expandIcon}をクリックすると、ドキュメントを表示、比較、フィルタリングできます。", - "discover.dscTour.stepExpand.expandIconAriaLabel": "展開アイコン", - "discover.dscTour.stepExpand.imageAltText": "展開アイコンをクリックして、ドキュメントのフィールドを検査、フィルタリングし、コンテキスト内でドキュメントを表示します。", - "discover.dscTour.stepExpand.title": "ドキュメントの展開", - "discover.dscTour.stepReorderColumns.description": "任意の順序まで列をドラッグします。", - "discover.dscTour.stepReorderColumns.imageAltText": "［列］ポップアップを使用して、任意の順序まで列をドラッグします。", - "discover.dscTour.stepReorderColumns.title": "テーブルの列を並べ替える", - "discover.dscTour.stepSort.description": "列見出しを使用して1つのフィールドを並べ替えるか、複数のフィールドに対してポップオーバーを使用します。", - "discover.dscTour.stepSort.imageAltText": "列見出しをクリックして、任意の並べ順を選択します。フィールドが並べ替えられたポップオーバーを使用して、複数のフィールドの並べ替えを調整します。", - "discover.dscTour.stepSort.title": "1つ以上のフィールドを並べ替えます", "discover.embeddable.inspectorRequestDataTitle": "データ", "discover.embeddable.inspectorRequestDescription": "このリクエストはElasticsearchにクエリーをかけ、検索データを取得します。", "discover.embeddable.search.displayName": "検索", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 4d5023b6cfd61..30d822480bdff 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -2407,11 +2407,6 @@ "discover.docExplorerCallout.headerMessage": "更好的浏览方式", "discover.docExplorerCallout.learnMore": "了解详情", "discover.docExplorerCallout.tryDocumentExplorer": "试用 Document Explorer", - "discover.docExplorerUpdateCallout.closeButtonAriaLabel": "关闭", - "discover.docExplorerUpdateCallout.description": "在文档表中添加相关字段，对列进行重新排序和排序，调整行大小等。", - "discover.docExplorerUpdateCallout.dismissButtonLabel": "关闭", - "discover.docExplorerUpdateCallout.takeTourButtonLabel": "学习教程", - "discover.docExplorerUpdateCallout.title": "仔细查看您的搜索结果", "discover.docTable.documentsNavigation": "文档导航", "discover.docTable.limitedSearchResultLabel": "仅限于 {resultCount} 个结果。优化您的搜索。", "discover.docTable.noResultsTitle": "找不到结果", @@ -2441,22 +2436,6 @@ "discover.documentsAriaLabel": "文档", "discover.docViews.table.scoreSortWarningTooltip": "要检索 _score 的值，必须按其筛选。", "discover.dropZoneTableLabel": "放置区域以将字段作为列添加到表中", - "discover.dscTour.stepAddFields.description": "单击 {plusIcon} 以添加您感兴趣的字段。", - "discover.dscTour.stepAddFields.imageAltText": "在可用字段列表中，单击加号图标将字段切换为文档表。", - "discover.dscTour.stepAddFields.title": "将字段添加到表中", - "discover.dscTour.stepChangeRowHeight.description": "调整行数以适应内容。", - "discover.dscTour.stepChangeRowHeight.imageAltText": "单击显示选项图标，调整行高以适应内容。", - "discover.dscTour.stepChangeRowHeight.title": "更改行高", - "discover.dscTour.stepExpand.description": "单击 {expandIcon} 以查看、比较和筛选文档。", - "discover.dscTour.stepExpand.expandIconAriaLabel": "展开图标", - "discover.dscTour.stepExpand.imageAltText": "单击展开图标以检查和筛选文档中的字段，并在上下文中查看文档。", - "discover.dscTour.stepExpand.title": "展开文档", - "discover.dscTour.stepReorderColumns.description": "拖动列以进行所需排序。", - "discover.dscTour.stepReorderColumns.imageAltText": "使用列弹出框按您想要的顺序拖动列。", - "discover.dscTour.stepReorderColumns.title": "对表列进行排序", - "discover.dscTour.stepSort.description": "使用列标题对单一字段进行排序，或使用弹出框对多个字段进行排序。", - "discover.dscTour.stepSort.imageAltText": "单击列标题并选择所需排序顺序。使用字段排序弹出框调整多字段排序。", - "discover.dscTour.stepSort.title": "对一个或多个字段排序", "discover.embeddable.inspectorRequestDataTitle": "数据", "discover.embeddable.inspectorRequestDescription": "此请求将查询 Elasticsearch 以获取搜索的数据。", "discover.embeddable.search.displayName": "搜索", diff --git a/x-pack/plugins/triggers_actions_ui/public/common/expression_items/components/closable_popover_title.tsx b/x-pack/plugins/triggers_actions_ui/public/common/expression_items/components/closable_popover_title.tsx index f6ebec92b4a57..60145943a83c1 100644 --- a/x-pack/plugins/triggers_actions_ui/public/common/expression_items/components/closable_popover_title.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/common/expression_items/components/closable_popover_title.tsx @@ -12,13 +12,18 @@ import { EuiPopoverTitle, EuiFlexGroup, EuiFlexItem, EuiButtonIcon } from '@elas interface ClosablePopoverTitleProps { children: JSX.Element; onClose: () => void; + dataTestSubj?: string; } -export const ClosablePopoverTitle = ({ children, onClose }: ClosablePopoverTitleProps) => { +export const ClosablePopoverTitle = ({ + children, + onClose, + dataTestSubj, +}: ClosablePopoverTitleProps) => { return ( - {children} + {children} { wrapper.find('[data-test-subj="thresholdPopover"]').last().simulate('click'); expect(wrapper.find('[data-test-subj="comparatorOptionsComboBox"]').exists()).toBeTruthy(); - expect(wrapper.find('[data-test-subj="alertThresholdInput"]').exists()).toBeTruthy(); + expect(wrapper.find('[data-test-subj="alertThresholdInput0"]').exists()).toBeTruthy(); wrapper - .find('[data-test-subj="alertThresholdInput"]') + .find('[data-test-subj="alertThresholdInput0"]') .last() .simulate('change', { target: { value: 1000 } }); expect(onChangeSelectedThreshold).toHaveBeenCalled(); @@ -145,21 +145,22 @@ describe('threshold expression', () => { wrapper.find('[data-test-subj="thresholdPopover"]').last().simulate('click'); expect(wrapper.find('[data-test-subj="comparatorOptionsComboBox"]').exists()).toBeTruthy(); - expect(wrapper.find('input[data-test-subj="alertThresholdInput"]').length).toEqual(1); + expect(wrapper.find('input[data-test-subj="alertThresholdInput0"]').length).toEqual(1); wrapper .find('[data-test-subj="comparatorOptionsComboBox"]') .last() .simulate('change', { target: { value: 'between' } }); wrapper.update(); - expect(wrapper.find('input[data-test-subj="alertThresholdInput"]').length).toEqual(2); + expect(wrapper.find('input[data-test-subj="alertThresholdInput0"]').length).toEqual(1); + expect(wrapper.find('input[data-test-subj="alertThresholdInput1"]').length).toEqual(1); wrapper .find('[data-test-subj="comparatorOptionsComboBox"]') .last() .simulate('change', { target: { value: '<' } }); wrapper.update(); - expect(wrapper.find('input[data-test-subj="alertThresholdInput"]').length).toEqual(1); + expect(wrapper.find('input[data-test-subj="alertThresholdInput0"]').length).toEqual(1); }); it('is valid when the threshold value is 0', () => { @@ -174,9 +175,9 @@ describe('threshold expression', () => { onChangeSelectedThresholdComparator={onChangeSelectedThresholdComparator} /> ); - expect(wrapper.find('[data-test-subj="alertThresholdInput"]')).toMatchInlineSnapshot(` + expect(wrapper.find('[data-test-subj="alertThresholdInput0"]')).toMatchInlineSnapshot(`
- setAlertThresholdPopoverOpen(false)}> + setAlertThresholdPopoverOpen(false)} + dataTestSubj="thresholdPopoverTitle" + > <>{comparators[comparator].text} @@ -154,7 +157,7 @@ export const ThresholdExpression = ({ error={errors[`threshold${i}`] as string[]} > 0 || isNil(threshold[i])} diff --git a/x-pack/test/api_integration/services/security_solution_endpoint_exceptions_api.gen.ts b/x-pack/test/api_integration/services/security_solution_endpoint_exceptions_api.gen.ts index 9f77c4575828a..505faf82fd681 100644 --- a/x-pack/test/api_integration/services/security_solution_endpoint_exceptions_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_endpoint_exceptions_api.gen.ts @@ -31,7 +31,7 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) return { /** - * Creates an endpoint list or does nothing if the list already exists + * Create an endpoint exception list, which groups endpoint exception list items. If an endpoint exception list already exists, an empty response is returned. */ createEndpointList() { return supertest @@ -40,6 +40,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Create an endpoint exception list item, and associate it with the endpoint exception list. + */ createEndpointListItem(props: CreateEndpointListItemProps) { return supertest .post('/api/endpoint_list/items') @@ -48,6 +51,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Delete an endpoint exception list item using the `id` or `item_id` field. + */ deleteEndpointListItem(props: DeleteEndpointListItemProps) { return supertest .delete('/api/endpoint_list/items') @@ -56,6 +62,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get a list of all endpoint exception list items. + */ findEndpointListItems(props: FindEndpointListItemsProps) { return supertest .get('/api/endpoint_list/items/_find') @@ -64,6 +73,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the details of an endpoint exception list item using the `id` or `item_id` field. + */ readEndpointListItem(props: ReadEndpointListItemProps) { return supertest .get('/api/endpoint_list/items') @@ -72,6 +84,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Update an endpoint exception list item using the `id` or `item_id` field. + */ updateEndpointListItem(props: UpdateEndpointListItemProps) { return supertest .put('/api/endpoint_list/items') diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap index a4d255613133e..9112ad20ad860 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap @@ -30,7 +30,6 @@ Object { "enabled": true, "name": "system-1", "namespace": "default", - "output_id": null, "package": Object { "name": "system", "requires_root": true, diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/create_standalone_api_key.ts b/x-pack/test/fleet_api_integration/apis/agent_policy/create_standalone_api_key.ts new file mode 100644 index 0000000000000..dd227d53911c6 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/create_standalone_api_key.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { SpaceTestApiClient } from '../space_awareness/api_helper'; +import { expectToRejectWithError } from '../space_awareness/helpers'; +import { setupTestUsers, testUsers } from '../test_users'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const supertest = getService('supertest'); + + describe('create standalone api key', function () { + skipIfNoDockerRegistry(providerContext); + + before(async () => { + await setupTestUsers(getService('security')); + }); + + describe('POST /internal/fleet/create_standalone_agent_api_key', () => { + it('should work with a user with the correct permissions', async () => { + const apiClient = new SpaceTestApiClient(supertest); + const res = await apiClient.postStandaloneApiKey('test'); + expect(res.item.name).to.eql('standalone_agent-test'); + }); + it('should return a 403 if the user cannot create the api key', async () => { + const apiClient = new SpaceTestApiClient(supertestWithoutAuth, { + username: testUsers.fleet_all_int_all.username, + password: testUsers.fleet_all_int_all.password, + }); + await expectToRejectWithError( + () => apiClient.postStandaloneApiKey('tata'), + /403 Forbidden Missing permissions to create standalone API key/ + ); + }); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/index.js b/x-pack/test/fleet_api_integration/apis/agent_policy/index.js index 66abbf8d6a5b3..9ae58b0089942 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/index.js +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/index.js @@ -12,5 +12,6 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./agent_policy_datastream_permissions')); loadTestFile(require.resolve('./privileges')); loadTestFile(require.resolve('./agent_policy_root_integrations')); + loadTestFile(require.resolve('./create_standalone_api_key')); }); } diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index 159c40926f4e7..ea50aaaf53eb8 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -198,7 +198,7 @@ export default function (providerContext: FtrProviderContext) { }, }) .expect(200); - expect(response.body.item.policy_id).to.eql(null); + expect(response.body.item.policy_id).to.eql(undefined); expect(response.body.item.policy_ids).to.eql([]); }); diff --git a/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts b/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts index 1de90ae3dcfaa..c8af244ba11b9 100644 --- a/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts +++ b/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts @@ -554,4 +554,19 @@ export class SpaceTestApiClient { return res; } + + async postStandaloneApiKey(name: string, spaceId?: string) { + const { body: res, statusCode } = await this.supertest + .post(`${this.getBaseUrl(spaceId)}/internal/fleet/create_standalone_agent_api_key`) + .auth(this.auth.username, this.auth.password) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '1') + .send({ name }); + + if (statusCode !== 200) { + throw new Error(`${statusCode} ${res?.error} ${res.message}`); + } + + return res; + } } diff --git a/x-pack/test/functional/services/observability/alerts/common.ts b/x-pack/test/functional/services/observability/alerts/common.ts index dcac33b952e3e..7f327664f1b71 100644 --- a/x-pack/test/functional/services/observability/alerts/common.ts +++ b/x-pack/test/functional/services/observability/alerts/common.ts @@ -6,9 +6,11 @@ */ import expect from '@kbn/expect'; +import { ToolingLog } from '@kbn/tooling-log'; import { chunk } from 'lodash'; import { ALERT_STATUS_ACTIVE, ALERT_STATUS_RECOVERED, AlertStatus } from '@kbn/rule-data-utils'; import { WebElementWrapper } from '@kbn/ftr-common-functional-ui-services'; +import { Agent as SuperTestAgent } from 'supertest'; import { FtrProviderContext } from '../../../ftr_provider_context'; // Based on the x-pack/test/functional/es_archives/observability/alerts archive. @@ -314,6 +316,69 @@ export function ObservabilityAlertsCommonProvider({ return value; }); + // Data view + const createDataView = async ({ + supertest, + id, + name, + title, + logger, + }: { + supertest: SuperTestAgent; + id: string; + name: string; + title: string; + logger: ToolingLog; + }) => { + const { body } = await supertest + .post(`/api/content_management/rpc/create`) + .set('kbn-xsrf', 'foo') + .send({ + contentTypeId: 'index-pattern', + data: { + fieldAttrs: '{}', + title, + timeFieldName: '@timestamp', + sourceFilters: '[]', + fields: '[]', + fieldFormatMap: '{}', + typeMeta: '{}', + runtimeFieldMap: '{}', + name, + }, + options: { id }, + version: 1, + }) + .expect(200); + + logger.debug(`Created data view: ${JSON.stringify(body)}`); + return body; + }; + + const deleteDataView = async ({ + supertest, + id, + logger, + }: { + supertest: SuperTestAgent; + id: string; + logger: ToolingLog; + }) => { + const { body } = await supertest + .post(`/api/content_management/rpc/delete`) + .set('kbn-xsrf', 'foo') + .send({ + contentTypeId: 'index-pattern', + id, + options: { force: true }, + version: 1, + }) + .expect(200); + + logger.debug(`Deleted data view id: ${id}`); + return body; + }; + return { getQueryBar, clearQueryBar, @@ -357,5 +422,7 @@ export function ObservabilityAlertsCommonProvider({ navigateToRulesLogsPage, navigateToRuleDetailsByRuleId, navigateToAlertDetails, + createDataView, + deleteDataView, }; } diff --git a/x-pack/test/functional/services/observability/alerts/rules_page.ts b/x-pack/test/functional/services/observability/alerts/rules_page.ts index 76f700f99b999..f5b16dc3914ab 100644 --- a/x-pack/test/functional/services/observability/alerts/rules_page.ts +++ b/x-pack/test/functional/services/observability/alerts/rules_page.ts @@ -7,6 +7,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; const METRIC_THRESHOLD_RULE_TYPE_SELECTOR = 'metrics.alert.threshold-SelectOption'; +const CUSTOM_THRESHOLD_RULE_TYPE_SELECTOR = 'observability.rules.custom_threshold-SelectOption'; export function ObservabilityAlertsRulesProvider({ getService }: FtrProviderContext) { const testSubjects = getService('testSubjects'); @@ -18,8 +19,9 @@ export function ObservabilityAlertsRulesProvider({ getService }: FtrProviderCont }; const clickCreateRuleButton = async () => { + await testSubjects.existOrFail('createRuleButton'); const createRuleButton = await testSubjects.find('createRuleButton'); - return createRuleButton.click(); + return await createRuleButton.click(); }; const clickRuleStatusDropDownMenu = async () => testSubjects.click('statusDropdown'); @@ -33,6 +35,7 @@ export function ObservabilityAlertsRulesProvider({ getService }: FtrProviderCont }; const clickOnInfrastructureCategory = async () => { + await testSubjects.existOrFail('ruleTypeModal'); const categories = await testSubjects.find('ruleTypeModal'); const category = await categories.findByCssSelector(`.euiFacetButton[title="Infrastructure"]`); await category.click(); @@ -43,6 +46,18 @@ export function ObservabilityAlertsRulesProvider({ getService }: FtrProviderCont await testSubjects.click(METRIC_THRESHOLD_RULE_TYPE_SELECTOR); }; + const clickOnObservabilityCategory = async () => { + await testSubjects.existOrFail('ruleTypeModal'); + const categories = await testSubjects.find('ruleTypeModal'); + const category = await categories.findByCssSelector(`.euiFacetButton[title="Observability"]`); + await category.click(); + }; + + const clickOnCustomThresholdRule = async () => { + await testSubjects.existOrFail(CUSTOM_THRESHOLD_RULE_TYPE_SELECTOR); + await testSubjects.click(CUSTOM_THRESHOLD_RULE_TYPE_SELECTOR); + }; + return { getManageRulesPageHref, clickCreateRuleButton, @@ -52,5 +67,7 @@ export function ObservabilityAlertsRulesProvider({ getService }: FtrProviderCont clickOnRuleInEventLogs, clickOnInfrastructureCategory, clickOnMetricThresholdRule, + clickOnObservabilityCategory, + clickOnCustomThresholdRule, }; } diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index f16689bb3d22f..8c176c61530b7 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -163,7 +163,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { return ruleName === alertName; }); await testSubjects.click('thresholdPopover'); - await testSubjects.setValue('alertThresholdInput', '1'); + await testSubjects.setValue('alertThresholdInput0', '1'); await testSubjects.click('forLastExpression'); await testSubjects.setValue('timeWindowSizeNumber', '30'); @@ -469,7 +469,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await filterBar.addFilter({ field: 'message.keyword', operation: 'is', value: 'msg-1' }); await testSubjects.click('thresholdPopover'); - await testSubjects.setValue('alertThresholdInput', '1'); + await testSubjects.setValue('alertThresholdInput0', '1'); await testSubjects.click('saveEditedRuleButton'); await PageObjects.header.waitUntilLoadingHasFinished(); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts index ffd1ad09c7ccd..e0e67066b4777 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts @@ -59,6 +59,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { messages, connectorId: 'does not exist', functions: [], + scope: 'all', }) .expect(404); }); @@ -87,6 +88,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { messages, connectorId, functions: [], + scope: 'all', }) .pipe(passThrough); @@ -144,6 +146,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { messages, connectorId, functions: [], + scope: 'all', }) .expect(200) .pipe(passThrough); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index 04d85d6dc282c..aaba5fbc7ba99 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -84,6 +84,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { connectorId, persist: true, screenContexts: params.screenContexts || [], + scope: 'all', }) .then((response) => resolve(response)) .catch((err) => reject(err)); @@ -136,6 +137,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { connectorId, persist: false, screenContexts: [], + scope: 'all', }) .pipe(passThrough); @@ -402,6 +404,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { connectorId, persist: true, screenContexts: [], + scope: 'observability', }, }, }) @@ -444,6 +447,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { persist: true, screenContexts: [], conversationId, + scope: 'observability', }, }, }) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts index a32c22abcf7aa..552b779d2c0aa 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -12,6 +12,7 @@ import { StreamingChatResponseEvent, } from '@kbn/observability-ai-assistant-plugin/common'; import { Readable } from 'stream'; +import { AssistantScope } from '@kbn/observability-ai-assistant-plugin/common/types'; import { CreateTest } from '../../../common/config'; function decodeEvents(body: Readable | string) { @@ -32,12 +33,14 @@ export async function invokeChatCompleteWithFunctionRequest({ connectorId, observabilityAIAssistantAPIClient, functionCall, + scope, }: { connectorId: string; observabilityAIAssistantAPIClient: Awaited< ReturnType >; functionCall: Message['message']['function_call']; + scope?: AssistantScope; }) { const { body } = await observabilityAIAssistantAPIClient .editorUser({ @@ -57,6 +60,7 @@ export async function invokeChatCompleteWithFunctionRequest({ connectorId, persist: false, screenContexts: [], + scope: scope || 'observability', }, }, }) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 4cad8079dc0b2..bf2eef14db553 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -249,6 +249,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { connectorId, persist: true, screenContexts: [], + scope: 'observability', }, }, }).expect(200); diff --git a/x-pack/test/observability_functional/apps/observability/index.ts b/x-pack/test/observability_functional/apps/observability/index.ts index 64636e79123d7..96256248e5d88 100644 --- a/x-pack/test/observability_functional/apps/observability/index.ts +++ b/x-pack/test/observability_functional/apps/observability/index.ts @@ -17,6 +17,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./pages/alerts/rule_stats')); loadTestFile(require.resolve('./pages/alerts/state_synchronization')); loadTestFile(require.resolve('./pages/alerts/table_storage')); + loadTestFile(require.resolve('./pages/alerts/custom_threshold')); loadTestFile(require.resolve('./pages/cases/case_details')); loadTestFile(require.resolve('./pages/overview/alert_table')); loadTestFile(require.resolve('./exploratory_view')); diff --git a/x-pack/test/observability_functional/apps/observability/pages/alerts/custom_threshold.ts b/x-pack/test/observability_functional/apps/observability/pages/alerts/custom_threshold.ts new file mode 100644 index 0000000000000..38d308a17e7b0 --- /dev/null +++ b/x-pack/test/observability_functional/apps/observability/pages/alerts/custom_threshold.ts @@ -0,0 +1,254 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Key } from 'selenium-webdriver'; +import expect from 'expect'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const testSubjects = getService('testSubjects'); + const kibanaServer = getService('kibanaServer'); + const supertest = getService('supertest'); + const find = getService('find'); + const logger = getService('log'); + const retry = getService('retry'); + + describe('Custom threshold rule', function () { + this.tags('includeFirefox'); + + const observability = getService('observability'); + const DATA_VIEW_1 = 'filebeat-*'; + const DATA_VIEW_1_ID = 'data-view-id_1'; + const DATA_VIEW_1_NAME = 'test-data-view-name_1'; + const DATA_VIEW_2 = 'metricbeat-*'; + const DATA_VIEW_2_ID = 'data-view-id_2'; + const DATA_VIEW_2_NAME = 'test-data-view-name_2'; + + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs'); + // create two data views + await observability.alerts.common.createDataView({ + supertest, + name: DATA_VIEW_1_NAME, + id: DATA_VIEW_1_ID, + title: DATA_VIEW_1, + logger, + }); + await observability.alerts.common.createDataView({ + supertest, + name: DATA_VIEW_2_NAME, + id: DATA_VIEW_2_ID, + title: DATA_VIEW_2, + logger, + }); + await observability.alerts.common.navigateToRulesPage(); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/infra/metrics_and_logs'); + // This also deletes the created data views + await kibanaServer.savedObjects.cleanStandardList(); + }); + + it('shows the custom threshold rule in the observability section', async () => { + await observability.alerts.rulesPage.clickCreateRuleButton(); + await observability.alerts.rulesPage.clickOnObservabilityCategory(); + await observability.alerts.rulesPage.clickOnCustomThresholdRule(); + }); + + it('can add name and tags', async () => { + await testSubjects.setValue('ruleNameInput', 'test custom threshold rule'); + await testSubjects.setValue('comboBoxSearchInput', 'tag1'); + }); + + it('can add data view', async () => { + // select data view + await testSubjects.click('selectDataViewExpression'); + await testSubjects.setValue('indexPattern-switcher--input', 'test-data-view-name_2'); + const dataViewExpression = await find.byCssSelector( + '[data-test-subj="indexPattern-switcher--input"]' + ); + await dataViewExpression.pressKeys(Key.ENTER); + await retry.waitFor('data view selection to happen', async () => { + const dataViewSelector = await testSubjects.find('selectDataViewExpression'); + return (await dataViewSelector.getVisibleText()) === 'DATA VIEW\ntest-data-view-name_2'; + }); + }); + + it('can select aggregation', async () => { + // select aggregation + await testSubjects.click('aggregationNameA'); + await testSubjects.click('aggregationTypeSelect'); + // assert all options are available + await find.byCssSelector('option[value="avg"]'); + await find.byCssSelector('option[value="min"]'); + await find.byCssSelector('option[value="max"]'); + await find.byCssSelector('option[value="sum"]'); + await find.byCssSelector('option[value="count"]'); + await find.byCssSelector('option[value="cardinality"]'); + await find.byCssSelector('option[value="p99"]'); + await find.byCssSelector('option[value="p95"]'); + await find.byCssSelector('option[value="rate"]'); + + // set first aggregation + await find.clickByCssSelector(`option[value="avg"]`); + const input1 = await find.byCssSelector('[data-test-subj="aggregationField"] input'); + await input1.type('metricset.rtt'); + await testSubjects.click('o11yClosablePopoverTitleButton'); + await retry.waitFor('first aggregation to happen', async () => { + const aggregationNameA = await testSubjects.find('aggregationNameA'); + return (await aggregationNameA.getVisibleText()) === 'AVERAGE\nmetricset.rtt'; + }); + await new Promise((r) => setTimeout(r, 1000)); + + // set second aggregation + await testSubjects.click('thresholdRuleCustomEquationEditorAddAggregationFieldButton'); + await testSubjects.click('aggregationNameB'); + await testSubjects.setValue('o11ySearchField', 'service.name : "opbeans-node"'); + await testSubjects.click('o11yClosablePopoverTitleButton'); + await retry.waitFor('first aggregation to happen', async () => { + const aggregationNameB = await testSubjects.find('aggregationNameB'); + return (await aggregationNameB.getVisibleText()) === 'COUNT\nservice.name : "opbeans-node"'; + }); + await new Promise((r) => setTimeout(r, 1000)); + }); + + it('can set custom equation', async () => { + // set custom equation + await testSubjects.click('customEquation'); + const customEquationField = await find.byCssSelector( + '[data-test-subj="thresholdRuleCustomEquationEditorFieldText"]' + ); + await customEquationField.click(); + await customEquationField.type('A - B'); + await testSubjects.click('o11yClosablePopoverTitleButton'); + await retry.waitFor('custom equation update to happen', async () => { + const customEquation = await testSubjects.find('customEquation'); + return (await customEquation.getVisibleText()) === 'EQUATION\nA - B'; + }); + await new Promise((r) => setTimeout(r, 1000)); + }); + + it('can set threshold', async () => { + // set threshold + await testSubjects.click('thresholdPopover'); + await testSubjects.click('comparatorOptionsComboBox'); + // assert all options are available + await find.byCssSelector('option[value=">="]'); + await find.byCssSelector('option[value="<="]'); + await find.byCssSelector('option[value=">"]'); + await find.byCssSelector('option[value="<"]'); + await find.byCssSelector('option[value="between"]'); + await find.byCssSelector('option[value="notBetween"]'); + // select an option + await find.clickByCssSelector(`option[value="notBetween"]`); + const thresholdField1 = await find.byCssSelector('[data-test-subj="alertThresholdInput0"]'); + await thresholdField1.click(); + await new Promise((r) => setTimeout(r, 1000)); + await thresholdField1.pressKeys(Key.BACK_SPACE); + await new Promise((r) => setTimeout(r, 1000)); + await thresholdField1.pressKeys(Key.BACK_SPACE); + await new Promise((r) => setTimeout(r, 1000)); + await thresholdField1.pressKeys(Key.BACK_SPACE); + await thresholdField1.type('200'); + const thresholdField2 = await find.byCssSelector('[data-test-subj="alertThresholdInput1"]'); + await thresholdField2.type('250'); + await find.clickByCssSelector('[aria-label="Close"]'); + await retry.waitFor('comparator selection to happen', async () => { + const customEquation = await testSubjects.find('thresholdPopover'); + return (await customEquation.getVisibleText()) === 'IS NOT BETWEEN\n200 AND 250'; + }); + }); + + it('can set equation label', async () => { + // set equation label + await testSubjects.setValue( + 'thresholdRuleCustomEquationEditorFieldTextLabel', + 'test equation' + ); + }); + + it('can set time range', async () => { + // set time range + await testSubjects.click('forLastExpression'); + await new Promise((r) => setTimeout(r, 1000)); + const timeRangeField = await find.byCssSelector('[data-test-subj="timeWindowSizeNumber"]'); + await timeRangeField.click(); + await new Promise((r) => setTimeout(r, 1000)); + await timeRangeField.pressKeys(Key.BACK_SPACE); + await timeRangeField.type('2'); + // assert all options are available + await testSubjects.click('timeWindowUnitSelect'); + await find.byCssSelector('option[value="s"]'); + await find.byCssSelector('option[value="m"]'); + await find.byCssSelector('option[value="h"]'); + await find.byCssSelector('option[value="d"]'); + // select an option + await new Promise((r) => setTimeout(r, 3000)); + await find.clickByCssSelector('[data-test-subj="timeWindowUnitSelect"] option[value="d"]'); + await find.clickByCssSelector('[aria-label="Close"]'); + }); + + it('can set groupby', async () => { + // set group by + const groupByField = await find.byCssSelector( + '[data-test-subj="thresholdRuleMetricsExplorer-groupBy"] [data-test-subj="comboBoxSearchInput"]' + ); + await groupByField.type('docker.container.name'); + }); + + it('can save the rule', async () => { + await testSubjects.click('saveRuleButton'); + await testSubjects.click('confirmModalConfirmButton'); + await find.byCssSelector('button[title="test custom threshold rule"]'); + }); + + it('saved the rule correctly', async () => { + const { body: rules } = await supertest.get('/internal/alerting/rules/_find'); + + expect(rules.data.length).toEqual(1); + expect(rules.data[0]).toEqual( + expect.objectContaining({ + name: 'test custom threshold rule', + tags: ['tag1'], + params: expect.objectContaining({ + alertOnGroupDisappear: false, + alertOnNoData: false, + criteria: [ + { + comparator: 'notBetween', + label: 'test equation', + equation: 'A - B', + metrics: [ + { + aggType: 'avg', + field: 'metricset.rtt', + name: 'A', + }, + { + aggType: 'count', + filter: 'service.name : "opbeans-node"', + name: 'B', + }, + ], + threshold: [200, 250], + timeSize: 2, + timeUnit: 'd', + }, + ], + groupBy: ['docker.container.name'], + searchConfiguration: { + index: 'data-view-id_2', + query: { query: '', language: 'kuery' }, + }, + }), + }) + ); + }); + }); +}; diff --git a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/index_threshold_rule.ts b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/index_threshold_rule.ts index 1155cf79a8e27..7d0b283a1f028 100644 --- a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/index_threshold_rule.ts +++ b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/index_threshold_rule.ts @@ -81,7 +81,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await nameInput1.click(); await testSubjects.click('thresholdPopover'); - await testSubjects.setValue('alertThresholdInput', '420000'); + await testSubjects.setValue('alertThresholdInput0', '420000'); await testSubjects.click('forLastExpression'); await testSubjects.setValue('timeWindowSizeNumber', '24'); await testSubjects.setValue('timeWindowUnitSelect', 'hours'); diff --git a/x-pack/test/security_api_integration/tests/session_idle/cleanup.ts b/x-pack/test/security_api_integration/tests/session_idle/cleanup.ts index 9af51a859befa..4a74e3938467e 100644 --- a/x-pack/test/security_api_integration/tests/session_idle/cleanup.ts +++ b/x-pack/test/security_api_integration/tests/session_idle/cleanup.ts @@ -18,6 +18,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertestWithoutAuth'); + const esSupertest = getService('esSupertest'); const es = getService('es'); const esDeleteAllIndices = getService('esDeleteAllIndices'); const config = getService('config'); @@ -93,9 +94,19 @@ export default function ({ getService }: FtrProviderContext) { }); } + async function addESDebugLoggingSettings() { + const addLogging = { + persistent: { + 'logger.org.elasticsearch.xpack.security.authc': 'debug', + }, + }; + await esSupertest.put('/_cluster/settings').send(addLogging).expect(200); + } + describe('Session Idle cleanup', () => { beforeEach(async () => { await es.cluster.health({ index: '.kibana_security_session*', wait_for_status: 'green' }); + await addESDebugLoggingSettings(); await esDeleteAllIndices('.kibana_security_session*'); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts index f08893fe4ded4..9c8d50631cc53 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts @@ -300,9 +300,11 @@ export default ({ getService }: FtrProviderContext) => { type: 'logs', dataset: 'elastic_agent.filebeat', }, - 'event.agent_id_status': 'verified', - 'event.ingested': '2022-03-23T16:50:28.994Z', - 'event.dataset': 'elastic_agent.filebeat', + event: { + agent_id_status: 'verified', + ingested: '2022-03-23T16:50:28.994Z', + dataset: 'elastic_agent.filebeat', + }, 'event.kind': 'signal', 'kibana.alert.ancestors': [ { @@ -466,9 +468,11 @@ export default ({ getService }: FtrProviderContext) => { type: 'logs', dataset: 'elastic_agent.filebeat', }, - 'event.agent_id_status': 'verified', - 'event.ingested': '2022-03-23T16:50:28.994Z', - 'event.dataset': 'elastic_agent.filebeat', + event: { + agent_id_status: 'verified', + ingested: '2022-03-23T16:50:28.994Z', + dataset: 'elastic_agent.filebeat', + }, 'event.kind': 'signal', 'kibana.alert.ancestors': [ { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts index cd67498c421c9..c0060912d6a52 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - describe('@serverless @serverlessQA @ess Rule exception operators for data type ip', () => { + describe('@serverless @serverlessQA @ess Rule exception operators for data type ip array', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/ip_as_array'); }); @@ -62,10 +62,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -86,9 +86,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -116,7 +116,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('should filter 3 ips if all 3 are set as exceptions', async () => { @@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); it('should filter a CIDR range of "127.0.0.1/30"', async () => { @@ -171,9 +171,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -193,7 +193,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); }); @@ -305,9 +305,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -327,7 +327,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('should filter 3 ips if all 3 are set as exceptions', async () => { @@ -346,7 +346,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); }); @@ -408,7 +408,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); }); @@ -458,9 +458,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -484,7 +484,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('will return 1 result if we have a list that includes all ips', async () => { @@ -513,7 +513,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); it('will return 2 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => { @@ -551,7 +551,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('will return 2 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.7', async () => { @@ -582,7 +582,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts index c4d50860ea5c2..016d83e1587be 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts @@ -33,7 +33,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - describe('@serverles @serverlessQA @ess Rule exception operators for data type keyword', () => { + describe('@serverless @serverlessQA @ess Rule exception operators for data type keyword array', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array'); }); @@ -65,10 +65,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -89,9 +89,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -119,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 keyword if all 3 are set as exceptions', async () => { @@ -154,7 +154,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -241,9 +241,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -263,7 +263,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 keyword if all 3 are set as exceptions', async () => { @@ -282,7 +282,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -344,7 +344,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -404,10 +404,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -442,9 +442,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -469,9 +469,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -495,7 +495,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('will return only the empty array for results if we have a list that includes all keyword', async () => { @@ -524,7 +524,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -653,9 +653,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -675,7 +675,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word five', null, 'word six', 'word seven']]); + expect(hits).to.eql([['word five', null, 'word six', 'word seven'], undefined]); }); it('should filter 3 keyword if all 3 are set as exceptions', async () => { @@ -694,7 +694,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts index bfc528cf6ad6b..e0271b7ddb934 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - describe('@serverless @serverlessQA @ess Rule exception operators for data type text', () => { + describe('@serverless @serverlessQA @ess Rule exception operators for data type text array', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/text_as_array'); }); @@ -62,10 +62,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -86,9 +86,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -116,7 +116,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 text if all 3 are set as exceptions', async () => { @@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -238,9 +238,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -260,7 +260,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 text if all 3 are set as exceptions', async () => { @@ -279,7 +279,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -341,7 +341,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -401,10 +401,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -439,9 +439,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -466,9 +466,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -492,7 +492,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('will return only the empty array for results if we have a list that includes all text', async () => { @@ -521,7 +521,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts index 608b7da833b6c..0b39a7287bacb 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts @@ -25,7 +25,7 @@ import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { Rule } from '@kbn/alerting-plugin/common'; import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import moment from 'moment'; -import { orderBy } from 'lodash'; +import { get, orderBy } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { @@ -2830,7 +2830,7 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]?._source?.destination).toEqual( expect.objectContaining({ domain: 'aaa.stage.11111111.hello' }) ); - expect(previewAlerts[0]?._source?.['event.dataset']).toEqual('network_traffic.tls'); + expect(get(previewAlerts[0]?._source, 'event.dataset')).toEqual('network_traffic.tls'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index 6a59d5244b88c..53b2843399c62 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -158,12 +158,12 @@ export default ({ getService }: FtrProviderContext) => { ecs: { version: '1.0.0-beta2', }, - ...flattenWithPrefix('event', { + event: { action: 'changed-audit-configuration', category: 'configuration', module: 'auditd', - kind: 'signal', - }), + }, + 'event.kind': 'signal', host: { architecture: 'x86_64', containerized: false, @@ -300,12 +300,11 @@ export default ({ getService }: FtrProviderContext) => { }, }, }, - ...flattenWithPrefix('event', { + event: { action: 'changed-audit-configuration', category: 'configuration', module: 'auditd', - kind: 'signal', - }), + }, service: { type: 'auditd', }, @@ -427,12 +426,11 @@ export default ({ getService }: FtrProviderContext) => { }, cloud: { instance: { id: '133551048' }, provider: 'digitalocean', region: 'ams3' }, ecs: { version: '1.0.0-beta2' }, - ...flattenWithPrefix('event', { + event: { action: 'changed-promiscuous-mode-on-device', category: 'anomoly', module: 'auditd', - kind: 'signal', - }), + }, host: { architecture: 'x86_64', containerized: false, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts index b688f53e288b1..0dd5a93bb9e60 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts @@ -282,12 +282,12 @@ export default ({ getService }: FtrProviderContext) => { ecs: { version: '1.0.0-beta2', }, - ...flattenWithPrefix('event', { + event: { action: 'error', category: 'user-login', module: 'auditd', - kind: 'signal', - }), + }, + 'event.kind': 'signal', host: { architecture: 'x86_64', containerized: false, @@ -464,12 +464,12 @@ export default ({ getService }: FtrProviderContext) => { ecs: { version: '1.0.0-beta2', }, - ...flattenWithPrefix('event', { + event: { action: 'error', category: 'user-login', module: 'auditd', - kind: 'signal', - }), + }, + 'event.kind': 'signal', host: { architecture: 'x86_64', containerized: false, @@ -749,7 +749,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -774,7 +773,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -826,7 +824,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threat.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -844,7 +841,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -901,7 +897,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threat.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -924,7 +919,6 @@ export default ({ getService }: FtrProviderContext) => { // threat.indicator.matched data). That's the case with the // first and third indicators matched, here. { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -943,7 +937,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1003,7 +996,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[0].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1024,7 +1016,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1042,7 +1033,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1063,7 +1053,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[1].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1172,7 +1161,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1197,7 +1185,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1274,7 +1261,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threat.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1292,7 +1278,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1390,7 +1375,6 @@ export default ({ getService }: FtrProviderContext) => { }>; assertContains(threatTerm.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1413,7 +1397,6 @@ export default ({ getService }: FtrProviderContext) => { // threat.indicator.matched data). That's the case with the // first and third indicators matched, here. { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1432,7 +1415,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1505,7 +1487,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[0].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1526,7 +1507,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1544,7 +1524,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1565,7 +1544,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[1].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts index dfb5b3492b17a..e274366e54aa7 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts @@ -160,14 +160,17 @@ export default ({ getService }: FtrProviderContext) => { name: expect.stringMatching(/(root|bob)/), terminal: 'pts/0', }, - 'event.action': 'user_login', - 'event.category': 'authentication', - 'event.dataset': 'login', + event: { + action: 'user_login', + category: 'authentication', + dataset: 'login', + + module: 'system', + origin: '/var/log/wtmp', + outcome: 'success', + type: 'authentication_success', + }, 'event.kind': 'signal', - 'event.module': 'system', - 'event.origin': '/var/log/wtmp', - 'event.outcome': 'success', - 'event.type': 'authentication_success', 'kibana.alert.original_time': '2019-02-19T20:42:08.230Z', 'kibana.alert.ancestors': [ { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts index 15ea0c02b6bc2..a4c59313389e3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts @@ -213,8 +213,7 @@ export default ({ getService }: FtrProviderContext) => { expect(errors).toEqual([]); - // event properties getting flattened - expect(alertSource).toHaveProperty(['event.created'], validDates); + expect(alertSource).toHaveProperty(['event', 'created'], validDates); }); // source threat.enrichments is keyword, ECS mapping for threat.enrichments is nested diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts index e97b9436612aa..79b118b3b3d94 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { get } from 'lodash'; import { EqlRuleCreateProps, ThresholdRuleCreateProps, @@ -73,7 +74,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', @@ -107,7 +108,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts index 5ef047ecd2de8..1be8274079663 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { get } from 'lodash'; import { EqlRuleCreateProps, @@ -60,7 +61,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', @@ -81,7 +82,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts index c0b42709847be..9d63a84a1cbe5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { get } from 'lodash'; import { EqlRuleCreateProps, ThresholdRuleCreateProps, @@ -75,7 +76,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 8, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', @@ -113,7 +114,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 8, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index 897c67c717c05..d7f9e6f7450e6 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -195,7 +195,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { return ruleName === alertName; }); await testSubjects.click('thresholdPopover'); - await testSubjects.setValue('alertThresholdInput', '1'); + await testSubjects.setValue('alertThresholdInput0', '1'); await testSubjects.click('forLastExpression'); await testSubjects.setValue('timeWindowSizeNumber', '30'); @@ -499,7 +499,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await filterBar.addFilter({ field: 'message.keyword', operation: 'is', value: 'msg-1' }); await testSubjects.click('thresholdPopover'); - await testSubjects.setValue('alertThresholdInput', '1'); + await testSubjects.setValue('alertThresholdInput0', '1'); await testSubjects.click('saveEditedRuleButton'); await PageObjects.header.waitUntilLoadingHasFinished(); diff --git a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts index 313dd001c4f4a..91b536c79c5e8 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts @@ -20,7 +20,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const testSubjects = getService('testSubjects'); const synthtrace = getService('svlLogsSynthtraceClient'); - describe('Onboarding Firehose Quickstart Flow', () => { + // Failing: See https://github.com/elastic/kibana/issues/193294 + describe.skip('Onboarding Firehose Quickstart Flow', () => { before(async () => { await PageObjects.svlCommonPage.loginAsAdmin(); // Onboarding requires admin role await PageObjects.common.navigateToUrlWithBrowserHistory( @@ -38,6 +39,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); beforeEach(async () => { + await (await testSubjects.find('createCloudFormationOptionAWSCLI')).click(); await testSubjects.existOrFail('observabilityOnboardingFirehoseCreateStackCommand'); }); @@ -51,10 +53,10 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { it('starts to monitor for incoming data after user leaves the page', async () => { await browser.execute(`window.dispatchEvent(new Event("blur"))`); - await testSubjects.isDisplayed('observabilityOnboardingAWSServiceList'); + await testSubjects.isDisplayed('observabilityOnboardingFirehoseProgressCallout'); }); - it('highlights an AWS service when data is detected', async () => { + it('shows an AWS service when data is detected', async () => { const DATASET = 'aws.vpcflow'; const AWS_SERVICE_ID = 'vpc-flow'; await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton'); @@ -79,11 +81,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }) ); - // Checking that an AWS service item is enabled after data is detected - await testSubjects - .find(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`) - .then((el) => el.findByTagName('button')) - .then((el) => el.isEnabled()); + // Checking that an AWS service item is visible after data is detected + await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`); }); }); } diff --git a/yarn.lock b/yarn.lock index a09aca8eff5ae..13852c94782c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -30422,6 +30422,18 @@ tinyqueue@^2.0.3: resolved "https://registry.yarnpkg.com/tinyqueue/-/tinyqueue-2.0.3.tgz#64d8492ebf39e7801d7bd34062e29b45b2035f08" integrity sha512-ppJZNDuKGgxzkHihX8v9v9G5f+18gzaTfrukGrq6ueg0lmH4nqVnA2IPG0AEH3jKEk2GRJCUhDoqpoiw3PHLBA== +tldts-core@^6.1.46: + version "6.1.46" + resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-6.1.46.tgz#062d64981ee83f934f875c178a97e42bcd13bef7" + integrity sha512-zA3ai/j4aFcmbqTvTONkSBuWs0Q4X4tJxa0gV9sp6kDbq5dAhQDSg0WUkReEm0fBAKAGNj+wPKCCsR8MYOYmwA== + +tldts@^6.1.32: + version "6.1.46" + resolved "https://registry.yarnpkg.com/tldts/-/tldts-6.1.46.tgz#0c3c4157efe732caeddd06eee6da891b26bd8a75" + integrity sha512-fw81lXV2CijkNrZAZvee7wegs+EOlTyIuVl/z4q6OUzZHQ1jGL2xQzKXq9geYf/1tzo9LZQLrkcko2m8HLh+rg== + dependencies: + tldts-core "^6.1.46" + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" @@ -30546,6 +30558,13 @@ tough-cookie@^4.1.2, tough-cookie@^4.1.3, tough-cookie@^4.1.4: universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-5.0.0.tgz#6b6518e2b5c070cf742d872ee0f4f92d69eac1af" + integrity sha512-FRKsF7cz96xIIeMZ82ehjC3xW2E+O2+v11udrDYewUbszngYhsGa8z6YUMMzO9QJZzzyd0nGGXnML/TReX6W8Q== + dependencies: + tldts "^6.1.32" + tr46@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09"