diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml index fbf0406f37be4..75909e7c21c46 100644 --- a/.buildkite/ftr_oblt_serverless_configs.yml +++ b/.buildkite/ftr_oblt_serverless_configs.yml @@ -6,6 +6,10 @@ disabled: - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts + # serverless config files that run deployment-agnostic tests + # Failing https://github.com/elastic/kibana/issues/195811 + - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts + defaultQueue: 'n2-4-spot' enabled: - x-pack/test_serverless/api_integration/test_suites/observability/config.ts @@ -25,5 +29,3 @@ enabled: - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts - # serverless config files that run deployment-agnostic tests - - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts diff --git a/.buildkite/ftr_search_serverless_configs.yml b/.buildkite/ftr_search_serverless_configs.yml index e6efee5860806..413558bffa0fe 100644 --- a/.buildkite/ftr_search_serverless_configs.yml +++ b/.buildkite/ftr_search_serverless_configs.yml @@ -1,6 +1,10 @@ disabled: # Base config files, only necessary to inform config finding script + # serverless config files that run deployment-agnostic tests + # Failing https://github.com/elastic/kibana/issues/195811 + - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts + defaultQueue: 'n2-4-spot' enabled: - x-pack/test_serverless/api_integration/test_suites/search/config.ts @@ -18,5 +22,3 @@ enabled: - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts - # serverless config files that run deployment-agnostic tests - - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index 6d42c030b2d4f..caf9fcc5ac92a 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -20,6 +20,10 @@ disabled: - x-pack/test_serverless/functional/config.base.ts - x-pack/test_serverless/shared/config.base.ts + # serverless config files that run deployment-agnostic tests + # Failing https://github.com/elastic/kibana/issues/195811 + - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts + defaultQueue: 'n2-4-spot' enabled: - x-pack/test_serverless/api_integration/test_suites/security/config.ts @@ -100,5 +104,3 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts - # serverless config files that run deployment-agnostic tests - - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts diff --git a/.buildkite/scripts/steps/cloud/build_and_deploy.sh b/.buildkite/scripts/steps/cloud/build_and_deploy.sh index 25e7d8fc631c9..220ab497aaf7b 100755 --- a/.buildkite/scripts/steps/cloud/build_and_deploy.sh +++ b/.buildkite/scripts/steps/cloud/build_and_deploy.sh @@ -51,7 +51,7 @@ fi if is_pr_with_label "ci:cloud-redeploy"; then echo "--- Shutdown Previous Deployment" CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id') - if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then + if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" == "null" ]; then echo "No deployment to remove" else echo "Shutting down previous deployment..." diff --git a/.buildkite/scripts/steps/serverless/deploy.sh b/.buildkite/scripts/steps/serverless/deploy.sh index d30723393dacd..cbbc6c6c664dd 100644 --- a/.buildkite/scripts/steps/serverless/deploy.sh +++ b/.buildkite/scripts/steps/serverless/deploy.sh @@ -56,7 +56,7 @@ deploy() { PROJECT_ID=$(jq -r '[.items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $PROJECT_EXISTS_LOGS) if is_pr_with_label "ci:project-redeploy"; then - if [ -z "${PROJECT_ID}" ]; then + if [ -z "${PROJECT_ID}" ] || [ "${PROJECT_ID}" == "null" ]; then echo "No project to remove" else echo "Shutting down previous project..." diff --git a/.github/updatecli/values.d/ironbank.yml b/.github/updatecli/values.d/ironbank.yml new file mode 100644 index 0000000000000..fd1134eda376a --- /dev/null +++ b/.github/updatecli/values.d/ironbank.yml @@ -0,0 +1,2 @@ +config: + - path: src/dev/build/tasks/os_packages/docker_generator/templates/ironbank \ No newline at end of file diff --git a/.github/updatecli/values.d/scm.yml b/.github/updatecli/values.d/scm.yml new file mode 100644 index 0000000000000..34d902fb389d5 --- /dev/null +++ b/.github/updatecli/values.d/scm.yml @@ -0,0 +1,11 @@ +scm: + enabled: true + owner: elastic + repository: kibana + branch: main + commitusingapi: true + # begin updatecli-compose policy values + user: kibanamachine + email: 42973632+kibanamachine@users.noreply.github.com + # end updatecli-compose policy values + diff --git a/.github/updatecli/values.d/updatecli-compose.yml b/.github/updatecli/values.d/updatecli-compose.yml new file mode 100644 index 0000000000000..02df609f2a30c --- /dev/null +++ b/.github/updatecli/values.d/updatecli-compose.yml @@ -0,0 +1,3 @@ +spec: + files: + - "updatecli-compose.yaml" \ No newline at end of file diff --git a/.github/workflows/updatecli-compose.yml b/.github/workflows/updatecli-compose.yml new file mode 100644 index 0000000000000..cbab42d3a63b1 --- /dev/null +++ b/.github/workflows/updatecli-compose.yml @@ -0,0 +1,38 @@ +--- +name: updatecli-compose + +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * *' + +permissions: + contents: read + +jobs: + compose: + runs-on: ubuntu-latest + permissions: + contents: write + packages: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + + - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: --experimental compose diff + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: --experimental compose apply + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/dev_docs/nav-kibana-dev.docnav.json b/dev_docs/nav-kibana-dev.docnav.json index 8b8cd64a44664..a7d696fc10574 100644 --- a/dev_docs/nav-kibana-dev.docnav.json +++ b/dev_docs/nav-kibana-dev.docnav.json @@ -278,6 +278,10 @@ { "id": "kibDevReactKibanaContext", "label": "Kibana React Contexts" + }, + { + "id": "kibDevDocsChromeRecentlyAccessed", + "label": "Recently Viewed" } ] }, diff --git a/dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.mdx b/dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.mdx new file mode 100644 index 0000000000000..cca466bcf1ac3 --- /dev/null +++ b/dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.mdx @@ -0,0 +1,66 @@ +--- +id: kibDevDocsChromeRecentlyAccessed +slug: /kibana-dev-docs/chrome/recently-accessed +title: Chrome Recently Viewed +description: How to use chrome's recently accessed service to add your links to the recently viewed list in the side navigation. +date: 2024-10-04 +tags: ['kibana', 'dev', 'contributor', 'chrome', 'navigation', 'shared-ux'] +--- + +## Introduction + +The service allows applications to register recently visited objects. These items are displayed in the "Recently Viewed" section of a side navigation menu, providing users with quick access to their previously visited resources. This service includes methods for adding, retrieving, and subscribing to the recently accessed history. + +![Recently viewed section in the sidenav](./chrome_recently_accessed.png) + +## Guidelines + +The service should be used thoughtfully to provide users with easy access to key resources they've interacted with. Unlike browser history, this feature is for important items that users may want to revisit. + +### DOs + +- Register important resources that users may want to revisit. Like a dashboard, a saved search, or another specific object. +- Update the link when the state of the current resource changes. For example, if a user changes the time range while on a dashboard, update the recently viewed link to reflect the latest viewed state where possible. See below for instructions on how to update the link when state changes. + +### DON'Ts + +- Don't register every page view. +- Don't register temporary or transient states as individual items. +- Prevent overloading. Keep the list focused on high-value resources. +- Don't add a recently viewed object without first speaking to relevant Product Managers. + +## Usage + +To register an item with the `ChromeRecentlyAccessed` service, provide a unique `id`, a `label`, and a `link`. The `id` is used to identify and deduplicate the item, the `label` is displayed in the "Recently Viewed" list and the `link` is used to navigate to the item when selected. + +```ts +const link = '/app/map/1234'; +const label = 'Map 1234'; +const id = 'map-1234'; + +coreStart.chrome.recentlyAccessed.add(link, label, id); +``` + +To update the link when state changes, add another item with the same `id`. This will replace the existing item in the "Recently Viewed" list. + +```ts +const link = '/app/map/1234'; +const label = 'Map 1234'; + +coreStart.chrome.recentlyAccessed.add(`/app/map/1234`, label, id); + +// User changes the time range and we want to update the link in the "Recently Viewed" list +coreStart.chrome.recentlyAccessed.add( + `/app/map/1234?timeRangeFrom=now-30m&timeRangeTo=now`, + label, + id +); +``` + +## Implementation details + +The services is based on package. This package provides a `RecentlyAccessedService` that uses browser local storage to manage records of recently accessed objects. Internally it implements the queue with a maximum length of 20 items. When the queue is full, the oldest item is removed. +Applications can create their own instance of `RecentlyAccessedService` to manage their own list of recently accessed items scoped to their application. + +- is a service available via `coreStart.chrome.recentlyAccessed` and should be used to add items to chrome's sidenav. +- is package that `ChromeRecentlyAccessed` is using internally and the package can be used to create your own instance and manage your own list of recently accessed items that is independent for chrome's sidenav. \ No newline at end of file diff --git a/dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.png b/dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.png new file mode 100644 index 0000000000000..41d3913b048a2 Binary files /dev/null and b/dev_docs/shared_ux/chrome_recently_accessed/chrome_recently_accessed.png differ diff --git a/dev_docs/shared_ux/shared_ux_landing.mdx b/dev_docs/shared_ux/shared_ux_landing.mdx index 4be8ad134be15..d96798eefa61f 100644 --- a/dev_docs/shared_ux/shared_ux_landing.mdx +++ b/dev_docs/shared_ux/shared_ux_landing.mdx @@ -66,5 +66,10 @@ layout: landing title: 'Kibana React Contexts', description: 'Learn how to use common React contexts in Kibana', }, + { + pageId: 'kibDevDocsChromeRecentlyAccessed', + title: 'Chrome Recently Viewed', + description: 'Learn how to add recently viewed items to the side navigation', + }, ]} /> diff --git a/docs/maps/connect-to-ems.asciidoc b/docs/maps/connect-to-ems.asciidoc index e41d544d64e4d..1ccdedb1da2a9 100644 --- a/docs/maps/connect-to-ems.asciidoc +++ b/docs/maps/connect-to-ems.asciidoc @@ -1,6 +1,6 @@ :ems: Elastic Maps Service :ems-docker-repo: docker.elastic.co/elastic-maps-service/elastic-maps-server -:ems-docker-image: {ems-docker-repo}:{version}-amd64 +:ems-docker-image: {ems-docker-repo}:{version} :ems-headers-url: https://deployment-host [[maps-connect-to-ems]] @@ -81,34 +81,53 @@ If you cannot connect to {ems} from the {kib} server or browser clients, and you {hosted-ems} is a self-managed version of {ems} offered as a Docker image that provides both the EMS basemaps and EMS boundaries. The image is bundled with basemaps up to zoom level 8. After connecting it to your {es} cluster for license validation, you have the option to download and configure a more detailed basemaps database. -You can use +docker pull+ to download the {hosted-ems} image from the Elastic Docker registry. - +. Pull the {hosted-ems} Docker image. ++ ifeval::["{release-state}"=="unreleased"] -Version {version} of {hosted-ems} has not yet been released, so no Docker image is currently available for this version. +WARNING: Version {version} of {hosted-ems} has not yet been released. +No Docker image is currently available for this version. endif::[] - -ifeval::["{release-state}"!="unreleased"] - ++ ["source","bash",subs="attributes"] ---------------------------------- docker pull {ems-docker-image} ---------------------------------- -Start {hosted-ems} and expose the default port `8080`: +. Optional: Install +https://docs.sigstore.dev/system_config/installation/[Cosign] for your +environment. Then use Cosign to verify the {es} image's signature. ++ +[source,sh,subs="attributes"] +---- +wget https://artifacts.elastic.co/cosign.pub +cosign verify --key cosign.pub {ems-docker-image} +---- ++ +The `cosign` command prints the check results and the signature payload in JSON format: ++ +[source,sh,subs="attributes"] +-------------------------------------------- +Verification for {ems-docker-image} -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - Existence of the claims in the transparency log was verified offline + - The signatures were verified against the specified public key +-------------------------------------------- + +. Start {hosted-ems} and expose the default port `8080`: ++ ["source","bash",subs="attributes"] ---------------------------------- docker run --rm --init --publish 8080:8080 \ {ems-docker-image} ---------------------------------- - ++ Once {hosted-ems} is running, follow instructions from the webpage at `localhost:8080` to define a configuration file and optionally download a more detailed basemaps database. - ++ [role="screenshot"] image::images/elastic-maps-server-instructions.png[Set-up instructions] -endif::[] - [float] [[elastic-maps-server-configuration]] ==== Configuration @@ -193,7 +212,6 @@ One way to configure {hosted-ems} is to provide `elastic-maps-server.yml` via bi ["source","yaml",subs="attributes"] -------------------------------------------- -version: '2' services: ems-server: image: {ems-docker-image} @@ -212,7 +230,6 @@ These variables can be set with +docker-compose+ like this: ["source","yaml",subs="attributes"] ---------------------------------------------------------- -version: '2' services: ems-server: image: {ems-docker-image} diff --git a/docs/maps/images/elastic-maps-server-instructions.png b/docs/maps/images/elastic-maps-server-instructions.png index 5c0b47ce8f49f..524ae2192b5e5 100644 Binary files a/docs/maps/images/elastic-maps-server-instructions.png and b/docs/maps/images/elastic-maps-server-instructions.png differ diff --git a/packages/kbn-apm-synthtrace/src/lib/logs/custom_logsdb_index_templates.ts b/packages/kbn-apm-synthtrace/src/lib/logs/custom_logsdb_index_templates.ts index a0b155444919e..3eadd3f3941de 100644 --- a/packages/kbn-apm-synthtrace/src/lib/logs/custom_logsdb_index_templates.ts +++ b/packages/kbn-apm-synthtrace/src/lib/logs/custom_logsdb_index_templates.ts @@ -25,6 +25,7 @@ export const indexTemplates: { template: { settings: { mode: 'logsdb', + default_pipeline: 'logs@default-pipeline', }, }, priority: 500, diff --git a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts index a6a64429f9b86..9673d1678132b 100644 --- a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts @@ -7,16 +7,20 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { Client } from '@elastic/elasticsearch'; +import { Client, estypes } from '@elastic/elasticsearch'; import { pipeline, Readable } from 'stream'; import { LogDocument } from '@kbn/apm-synthtrace-client/src/lib/logs'; -import { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { IngestProcessorContainer, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { ValuesType } from 'utility-types'; import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; import { getSerializeTransform } from '../shared/get_serialize_transform'; import { Logger } from '../utils/create_logger'; import { indexTemplates, IndexTemplateName } from './custom_logsdb_index_templates'; import { getRoutingTransform } from '../shared/data_stream_get_routing_transform'; +export const LogsIndex = 'logs'; +export const LogsCustom = 'logs@custom'; + export type LogsSynthtraceEsClientOptions = Omit; export class LogsSynthtraceEsClient extends SynthtraceEsClient { @@ -60,6 +64,47 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { this.logger.error(`Index creation failed: ${index} - ${err.message}`); } } + + async updateIndexTemplate( + indexName: string, + modify: ( + template: ValuesType< + estypes.IndicesGetIndexTemplateResponse['index_templates'] + >['index_template'] + ) => estypes.IndicesPutIndexTemplateRequest + ) { + try { + const response = await this.client.indices.getIndexTemplate({ + name: indexName, + }); + + await Promise.all( + response.index_templates.map((template) => { + return this.client.indices.putIndexTemplate({ + ...modify(template.index_template), + name: template.name, + }); + }) + ); + + this.logger.info(`Updated ${indexName} index template`); + } catch (err) { + this.logger.error(`Update index template failed: ${indexName} - ${err.message}`); + } + } + + async createCustomPipeline(processors: IngestProcessorContainer[]) { + try { + this.client.ingest.putPipeline({ + id: LogsCustom, + processors, + version: 1, + }); + this.logger.info(`Custom pipeline created: ${LogsCustom}`); + } catch (err) { + this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`); + } + } } function logsPipeline() { diff --git a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts index 47dd4ffd2652f..b3e41bbdd4e28 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts @@ -16,12 +16,10 @@ import { getCluster, getCloudRegion, getCloudProvider, + MORE_THAN_1024_CHARS, } from './helpers/logs_mock_data'; import { parseLogsScenarioOpts } from './helpers/logs_scenario_opts_parser'; -const MORE_THAN_1024_CHARS = - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; - // Logs Data logic const MESSAGE_LOG_LEVELS = [ { message: 'A simple log', level: 'info' }, diff --git a/packages/kbn-apm-synthtrace/src/scenarios/degraded_synthetics_monitors.ts b/packages/kbn-apm-synthtrace/src/scenarios/degraded_synthetics_monitors.ts index c61fecd8b7109..6e00bfd0abf15 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/degraded_synthetics_monitors.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/degraded_synthetics_monitors.ts @@ -14,12 +14,9 @@ import { } from '@kbn/apm-synthtrace-client'; import { Scenario } from '../cli/scenario'; import { withClient } from '../lib/utils/with_client'; -import { getIpAddress } from './helpers/logs_mock_data'; +import { MORE_THAN_1024_CHARS, getIpAddress } from './helpers/logs_mock_data'; import { getAtIndexOrRandom } from './helpers/get_at_index_or_random'; -const MORE_THAN_1024_CHARS = - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; - const MONITOR_NAMES = Array(4) .fill(null) .map((_, idx) => `synth-monitor-${idx}`); diff --git a/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts new file mode 100644 index 0000000000000..91ddedac270b5 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { LogDocument, log, generateShortId, generateLongId } from '@kbn/apm-synthtrace-client'; +import { merge } from 'lodash'; +import { Scenario } from '../cli/scenario'; +import { IndexTemplateName } from '../lib/logs/custom_logsdb_index_templates'; +import { withClient } from '../lib/utils/with_client'; +import { + getServiceName, + getCluster, + getCloudRegion, + getCloudProvider, + MORE_THAN_1024_CHARS, +} from './helpers/logs_mock_data'; +import { parseLogsScenarioOpts } from './helpers/logs_scenario_opts_parser'; +import { LogsIndex } from '../lib/logs/logs_synthtrace_es_client'; + +const processors = [ + { + script: { + tag: 'normalize log level', + lang: 'painless', + source: ` + String level = ctx['log.level']; + if ('0'.equals(level)) { + ctx['log.level'] = 'info'; + } else if ('1'.equals(level)) { + ctx['log.level'] = 'debug'; + } else if ('2'.equals(level)) { + ctx['log.level'] = 'warning'; + } else if ('3'.equals(level)) { + ctx['log.level'] = 'error'; + } else { + throw new Exception("Not a valid log level"); + } + `, + }, + }, +]; + +// Logs Data logic +const MESSAGE_LOG_LEVELS = [ + { message: 'A simple log', level: '0' }, + { + message: 'Another log message', + level: '1', + }, + { + message: 'A log message generated from a warning', + level: '2', + }, + { message: 'Error with certificate: "ca_trusted_fingerprint"', level: '3' }, +]; + +const scenario: Scenario = async (runOptions) => { + const { isLogsDb } = parseLogsScenarioOpts(runOptions.scenarioOpts); + return { + bootstrap: async ({ logsEsClient }) => { + await logsEsClient.createCustomPipeline(processors); + if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); + + await logsEsClient.updateIndexTemplate( + isLogsDb ? IndexTemplateName.LogsDb : LogsIndex, + (template) => { + const next = { + name: LogsIndex, + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + }, + generate: ({ range, clients: { logsEsClient } }) => { + const { logger } = runOptions; + + const constructLogsCommonData = () => { + const index = Math.floor(Math.random() * 3); + const serviceName = getServiceName(index); + const logMessage = MESSAGE_LOG_LEVELS[index]; + const { clusterId, clusterName } = getCluster(index); + const cloudRegion = getCloudRegion(index); + + const commonLongEntryFields: LogDocument = { + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + 'orchestrator.cluster.name': clusterName, + 'orchestrator.cluster.id': clusterId, + 'orchestrator.resource.id': generateShortId(), + 'cloud.provider': getCloudProvider(), + 'cloud.region': cloudRegion, + 'cloud.availability_zone': `${cloudRegion}a`, + 'cloud.project.id': generateShortId(), + 'cloud.instance.id': generateShortId(), + 'log.file.path': `/logs/${generateLongId()}/error.txt`, + }; + + return { + index, + serviceName, + logMessage, + cloudRegion, + commonLongEntryFields, + }; + }; + + const datasetSynth1Logs = (timestamp: number) => { + const { + serviceName, + logMessage: { level, message }, + commonLongEntryFields, + } = constructLogsCommonData(); + + return log + .create({ isLogsDb }) + .dataset('synth.1') + .message(message) + .logLevel(level) + .service(serviceName) + .defaults(commonLongEntryFields) + .timestamp(timestamp); + }; + + const datasetSynth2Logs = (i: number, timestamp: number) => { + const { + serviceName, + logMessage: { level, message }, + commonLongEntryFields, + } = constructLogsCommonData(); + const isFailed = i % 60 === 0; + return log + .create({ isLogsDb }) + .dataset('synth.2') + .message(message) + .logLevel(isFailed ? '4' : level) // "script_exception": Not a valid log level + .service(serviceName) + .defaults(commonLongEntryFields) + .timestamp(timestamp); + }; + + const datasetSynth3Logs = (i: number, timestamp: number) => { + const { + serviceName, + logMessage: { level, message }, + cloudRegion, + commonLongEntryFields, + } = constructLogsCommonData(); + const isMalformed = i % 10 === 0; + const isFailed = i % 80 === 0; + return log + .create({ isLogsDb }) + .dataset('synth.3') + .message(message) + .logLevel(isFailed ? '5' : level) // "script_exception": Not a valid log level + .service(serviceName) + .defaults({ + ...commonLongEntryFields, + 'cloud.availability_zone': isMalformed + ? MORE_THAN_1024_CHARS // "ignore_above": 1024 in mapping + : `${cloudRegion}a`, + }) + .timestamp(timestamp); + }; + + const logs = range + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(200) + .fill(0) + .flatMap((_, index) => [ + datasetSynth1Logs(timestamp), + datasetSynth2Logs(index, timestamp), + datasetSynth3Logs(index, timestamp), + ]); + }); + + return withClient( + logsEsClient, + logger.perf('generating_logs', () => logs) + ); + }, + }; +}; + +export default scenario; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/helpers/logs_mock_data.ts b/packages/kbn-apm-synthtrace/src/scenarios/helpers/logs_mock_data.ts index e974528f16a80..5f3cbd5f054dd 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/helpers/logs_mock_data.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/helpers/logs_mock_data.ts @@ -59,6 +59,9 @@ const SERVICE_NAMES = Array(3) .fill(null) .map((_, idx) => `synth-service-${idx}`); +export const MORE_THAN_1024_CHARS = + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; + // Functions to get random elements export const getCluster = (index?: number) => getAtIndexOrRandom(CLUSTER, index); export const getIpAddress = (index?: number) => getAtIndexOrRandom(IP_ADDRESSES, index); diff --git a/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts b/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts index 8a6bdf409a573..6dac3fc9f3226 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts @@ -8,21 +8,22 @@ */ import { - log, - LogDocument, + ApmFields, InfraDocument, - apm, Instance, - infra, - ApmFields, + LogDocument, + apm, generateShortId, + infra, + log, } from '@kbn/apm-synthtrace-client'; import { Scenario } from '../cli/scenario'; +import { IndexTemplateName } from '../lib/logs/custom_logsdb_index_templates'; import { Logger } from '../lib/utils/create_logger'; -import { withClient } from '../lib/utils/with_client'; import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment'; +import { withClient } from '../lib/utils/with_client'; +import { MORE_THAN_1024_CHARS } from './helpers/logs_mock_data'; import { parseLogsScenarioOpts, parseStringToBoolean } from './helpers/logs_scenario_opts_parser'; -import { IndexTemplateName } from '../lib/logs/custom_logsdb_index_templates'; const ENVIRONMENT = getSynthtraceEnvironment(__filename); @@ -475,6 +476,3 @@ const DATASETS = [ ]; const LOG_LEVELS = ['info', 'error', 'warn', 'debug']; - -const MORE_THAN_1024_CHARS = - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts index 3c1fdc5131395..08d914c1017dd 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts @@ -7,19 +7,20 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { LogDocument, log, generateShortId, generateLongId } from '@kbn/apm-synthtrace-client'; +import { LogDocument, generateLongId, generateShortId, log } from '@kbn/apm-synthtrace-client'; import moment from 'moment'; import { Scenario } from '../cli/scenario'; import { IndexTemplateName } from '../lib/logs/custom_logsdb_index_templates'; import { withClient } from '../lib/utils/with_client'; import { - getServiceName, - getGeoCoordinate, - getIpAddress, - getCluster, + MORE_THAN_1024_CHARS, + getAgentName, getCloudProvider, getCloudRegion, - getAgentName, + getCluster, + getGeoCoordinate, + getIpAddress, + getServiceName, } from './helpers/logs_mock_data'; import { parseLogsScenarioOpts } from './helpers/logs_scenario_opts_parser'; @@ -30,9 +31,6 @@ const MESSAGE_LOG_LEVELS = [ { message: 'Error with certificate: "ca_trusted_fingerprint"', level: 'error' }, ]; -const MORE_THAN_1024_CHARS = - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; - const scenario: Scenario = async (runOptions) => { const { isLogsDb } = parseLogsScenarioOpts(runOptions.scenarioOpts); diff --git a/packages/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts b/packages/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts index 21330d0fea3b1..2dfe239ce5b88 100644 --- a/packages/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts +++ b/packages/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts @@ -593,6 +593,85 @@ ROW (asdf + asdf)::string, 1.2::string, "1234"::integer, (12321342134 + 23412341 - "aaaaaaaaaaa")::boolean`); }); }); + + describe('list literals', () => { + describe('numeric', () => { + test('wraps long list literals one line', () => { + const query = + 'ROW [1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890]'; + const text = reprint(query).text; + + expect('\n' + text).toBe(` +ROW + [1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890]`); + }); + + test('wraps long list literals to multiple lines one line', () => { + const query = `ROW [1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890]`; + const text = reprint(query).text; + + expect('\n' + text).toBe(` +ROW + [1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, 1234567890, + 1234567890, 1234567890, 1234567890]`); + }); + + test('breaks very long values one-per-line', () => { + const query = `ROW fn1(fn2(fn3(fn4(fn5(fn6(fn7(fn8([1234567890, 1234567890, 1234567890, 1234567890, 1234567890]))))))))`; + const text = reprint(query, { wrap: 40 }).text; + + expect('\n' + text).toBe(` +ROW + FN1( + FN2( + FN3( + FN4( + FN5( + FN6( + FN7( + FN8( + [ + 1234567890, + 1234567890, + 1234567890, + 1234567890, + 1234567890]))))))))`); + }); + }); + + describe('string', () => { + test('wraps long list literals one line', () => { + const query = + 'ROW ["some text", "another text", "one more text literal", "and another one", "and one more", "and one more", "and one more", "and one more", "and one more"]'; + const text = reprint(query).text; + + expect('\n' + text).toBe(` +ROW + ["some text", "another text", "one more text literal", "and another one", + "and one more", "and one more", "and one more", "and one more", + "and one more"]`); + }); + + test('can break very long strings per line', () => { + const query = + 'ROW ["..............................................", "..............................................", ".............................................."]'; + const text = reprint(query).text; + + expect('\n' + text).toBe(` +ROW + [ + "..............................................", + "..............................................", + ".............................................."]`); + }); + }); + }); }); test.todo('Idempotence on multiple times pretty printing'); diff --git a/packages/kbn-esql-ast/src/pretty_print/wrapping_pretty_printer.ts b/packages/kbn-esql-ast/src/pretty_print/wrapping_pretty_printer.ts index fde7f60a1dba5..91f65a389f0c3 100644 --- a/packages/kbn-esql-ast/src/pretty_print/wrapping_pretty_printer.ts +++ b/packages/kbn-esql-ast/src/pretty_print/wrapping_pretty_printer.ts @@ -15,9 +15,10 @@ import { CommandVisitorContext, ExpressionVisitorContext, FunctionCallExpressionVisitorContext, + ListLiteralExpressionVisitorContext, Visitor, } from '../visitor'; -import { singleItems } from '../visitor/utils'; +import { children, singleItems } from '../visitor/utils'; import { BasicPrettyPrinter, BasicPrettyPrinterOptions } from './basic_pretty_printer'; import { getPrettyPrintStats } from './helpers'; import { LeafPrinter } from './leaf_printer'; @@ -235,7 +236,11 @@ export class WrappingPrettyPrinter { } private printArguments( - ctx: CommandVisitorContext | CommandOptionVisitorContext | FunctionCallExpressionVisitorContext, + ctx: + | CommandVisitorContext + | CommandOptionVisitorContext + | FunctionCallExpressionVisitorContext + | ListLiteralExpressionVisitorContext, inp: Input ) { let txt = ''; @@ -247,7 +252,7 @@ export class WrappingPrettyPrinter { let remainingCurrentLine = inp.remaining; let oneArgumentPerLine = false; - for (const child of singleItems(ctx.node.args)) { + for (const child of children(ctx.node)) { if (getPrettyPrintStats(child).hasLineBreakingDecorations) { oneArgumentPerLine = true; break; @@ -489,13 +494,11 @@ export class WrappingPrettyPrinter { }) .on('visitListLiteralExpression', (ctx, inp: Input): Output => { - let elements = ''; - - for (const out of ctx.visitElements(inp)) { - elements += (elements ? ', ' : '') + out.txt; - } - - const formatted = `[${elements}]${inp.suffix ?? ''}`; + const args = this.printArguments(ctx, { + indent: inp.indent, + remaining: inp.remaining - 1, + }); + const formatted = `[${args.txt}]${inp.suffix ?? ''}`; const { txt, indented } = this.decorateWithComments(inp.indent, ctx.node, formatted); return { txt, indented }; diff --git a/packages/kbn-esql-ast/src/types.ts b/packages/kbn-esql-ast/src/types.ts index 0ca48b2326f7d..1bac6e0cff5b3 100644 --- a/packages/kbn-esql-ast/src/types.ts +++ b/packages/kbn-esql-ast/src/types.ts @@ -40,6 +40,7 @@ export type ESQLAstField = ESQLFunction | ESQLColumn; export type ESQLAstItem = ESQLSingleAstItem | ESQLAstItem[]; export type ESQLAstNodeWithArgs = ESQLCommand | ESQLCommandOption | ESQLFunction; +export type ESQLAstNodeWithChildren = ESQLAstNodeWithArgs | ESQLList; /** * *Proper* are nodes which are objects with `type` property, once we get rid diff --git a/packages/kbn-esql-ast/src/visitor/contexts.ts b/packages/kbn-esql-ast/src/visitor/contexts.ts index 0f637962b7ddd..4b4f04fdca4bb 100644 --- a/packages/kbn-esql-ast/src/visitor/contexts.ts +++ b/packages/kbn-esql-ast/src/visitor/contexts.ts @@ -12,11 +12,12 @@ // and makes it harder to understand the code structure. import { type GlobalVisitorContext, SharedData } from './global_visitor_context'; -import { firstItem, singleItems } from './utils'; +import { children, firstItem, singleItems } from './utils'; import type { ESQLAstCommand, ESQLAstItem, ESQLAstNodeWithArgs, + ESQLAstNodeWithChildren, ESQLAstRenameExpression, ESQLColumn, ESQLCommandOption, @@ -47,6 +48,11 @@ import { Builder } from '../builder'; const isNodeWithArgs = (x: unknown): x is ESQLAstNodeWithArgs => !!x && typeof x === 'object' && Array.isArray((x as any).args); +const isNodeWithChildren = (x: unknown): x is ESQLAstNodeWithChildren => + !!x && + typeof x === 'object' && + (Array.isArray((x as any).args) || Array.isArray((x as any).values)); + export class VisitorContext< Methods extends VisitorMethods = VisitorMethods, Data extends SharedData = SharedData, @@ -99,13 +105,13 @@ export class VisitorContext< public arguments(): ESQLAstExpressionNode[] { const node = this.node; - if (!isNodeWithArgs(node)) { + if (!isNodeWithChildren(node)) { return []; } const args: ESQLAstExpressionNode[] = []; - for (const arg of singleItems(node.args)) { + for (const arg of children(node)) { args.push(arg); } diff --git a/packages/kbn-esql-ast/src/visitor/utils.ts b/packages/kbn-esql-ast/src/visitor/utils.ts index 2e54a89c2bf52..0dc95b73cf9d7 100644 --- a/packages/kbn-esql-ast/src/visitor/utils.ts +++ b/packages/kbn-esql-ast/src/visitor/utils.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { ESQLAstItem, ESQLSingleAstItem } from '../types'; +import { ESQLAstItem, ESQLProperNode, ESQLSingleAstItem } from '../types'; /** * Normalizes AST "item" list to only contain *single* items. @@ -48,3 +48,32 @@ export const lastItem = (items: ESQLAstItem[]): ESQLSingleAstItem | undefined => if (Array.isArray(last)) return lastItem(last as ESQLAstItem[]); return last as ESQLSingleAstItem; }; + +export function* children(node: ESQLProperNode): Iterable { + switch (node.type) { + case 'function': + case 'command': + case 'option': { + for (const arg of singleItems(node.args)) { + yield arg; + } + break; + } + case 'list': { + for (const item of singleItems(node.values)) { + yield item; + } + break; + } + case 'inlineCast': { + if (Array.isArray(node.value)) { + for (const item of singleItems(node.value)) { + yield item; + } + } else { + yield node.value; + } + break; + } + } +} diff --git a/packages/kbn-esql-utils/index.ts b/packages/kbn-esql-utils/index.ts index 223181f2bd154..333557964d873 100644 --- a/packages/kbn-esql-utils/index.ts +++ b/packages/kbn-esql-utils/index.ts @@ -29,6 +29,7 @@ export { isQueryWrappedByPipes, retrieveMetadataColumns, getQueryColumnsFromESQLQuery, + isESQLColumnSortable, TextBasedLanguages, } from './src'; diff --git a/packages/kbn-esql-utils/src/index.ts b/packages/kbn-esql-utils/src/index.ts index e36283c7a9238..3b3228e7a2a4a 100644 --- a/packages/kbn-esql-utils/src/index.ts +++ b/packages/kbn-esql-utils/src/index.ts @@ -31,3 +31,4 @@ export { getStartEndParams, hasStartEndParams, } from './utils/run_query'; +export { isESQLColumnSortable } from './utils/esql_fields_utils'; diff --git a/packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts b/packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts new file mode 100644 index 0000000000000..ef8a24e686bd6 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; +import { isESQLColumnSortable } from './esql_fields_utils'; + +describe('esql fields helpers', () => { + describe('isESQLColumnSortable', () => { + it('returns false for geo fields', () => { + const geoField = { + id: 'geo.coordinates', + name: 'geo.coordinates', + meta: { + type: 'geo_point', + esType: 'geo_point', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(geoField)).toBeFalsy(); + }); + + it('returns false for source fields', () => { + const sourceField = { + id: '_source', + name: '_source', + meta: { + type: '_source', + esType: '_source', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(sourceField)).toBeFalsy(); + }); + + it('returns false for counter fields', () => { + const tsdbField = { + id: 'tsbd_counter', + name: 'tsbd_counter', + meta: { + type: 'number', + esType: 'counter_long', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(tsdbField)).toBeFalsy(); + }); + + it('returns true for everything else', () => { + const keywordField = { + id: 'sortable', + name: 'sortable', + meta: { + type: 'string', + esType: 'keyword', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(keywordField)).toBeTruthy(); + }); + }); +}); diff --git a/packages/kbn-esql-utils/src/utils/esql_fields_utils.ts b/packages/kbn-esql-utils/src/utils/esql_fields_utils.ts new file mode 100644 index 0000000000000..f5a0fe7b81340 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/esql_fields_utils.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; + +const SPATIAL_FIELDS = ['geo_point', 'geo_shape', 'point', 'shape']; +const SOURCE_FIELD = '_source'; +const TSDB_COUNTER_FIELDS_PREFIX = 'counter_'; + +/** + * Check if a column is sortable. + * + * @param column The DatatableColumn of the field. + * @returns True if the column is sortable, false otherwise. + */ + +export const isESQLColumnSortable = (column: DatatableColumn): boolean => { + // We don't allow sorting on spatial fields + if (SPATIAL_FIELDS.includes(column.meta?.type)) { + return false; + } + + // we don't allow sorting on the _source field + if (column.meta?.type === SOURCE_FIELD) { + return false; + } + + // we don't allow sorting on tsdb counter fields + if (column.meta?.esType && column.meta?.esType?.indexOf(TSDB_COUNTER_FIELDS_PREFIX) !== -1) { + return false; + } + + return true; +}; diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts index 84779f1dd36b5..a0a4a359c5ff6 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts @@ -387,6 +387,23 @@ describe('autocomplete', () => { '```````````````````````````````round(doubleField) + 1```````````````` + 1```````` + 1```` + 1`` + 1`', ] ); + + it('should not suggest already-used fields and variables', async () => { + const { suggest: suggestTest } = await setup(); + const getSuggestions = async (query: string) => + (await suggestTest(query)).map((value) => value.text); + + expect(await getSuggestions('from a_index | EVAL foo = 1 | KEEP /')).toContain('foo'); + expect(await getSuggestions('from a_index | EVAL foo = 1 | KEEP foo, /')).not.toContain( + 'foo' + ); + expect(await getSuggestions('from a_index | EVAL foo = 1 | KEEP /')).toContain( + 'doubleField' + ); + expect( + await getSuggestions('from a_index | EVAL foo = 1 | KEEP doubleField, /') + ).not.toContain('doubleField'); + }); }); } @@ -1111,11 +1128,14 @@ describe('autocomplete', () => { ]); }); - describe('KEEP ', () => { + describe.each(['KEEP', 'DROP'])('%s ', (commandName) => { // KEEP field - testSuggestions('FROM a | KEEP /', getFieldNamesByType('any').map(attachTriggerCommand)); testSuggestions( - 'FROM a | KEEP d/', + `FROM a | ${commandName} /`, + getFieldNamesByType('any').map(attachTriggerCommand) + ); + testSuggestions( + `FROM a | ${commandName} d/`, getFieldNamesByType('any') .map((text) => ({ text, @@ -1124,11 +1144,11 @@ describe('autocomplete', () => { .map(attachTriggerCommand) ); testSuggestions( - 'FROM a | KEEP doubleFiel/', + `FROM a | ${commandName} doubleFiel/`, getFieldNamesByType('any').map(attachTriggerCommand) ); testSuggestions( - 'FROM a | KEEP doubleField/', + `FROM a | ${commandName} doubleField/`, ['doubleField, ', 'doubleField | '] .map((text) => ({ text, @@ -1141,7 +1161,7 @@ describe('autocomplete', () => { // Let's get funky with the field names testSuggestions( - 'FROM a | KEEP @timestamp/', + `FROM a | ${commandName} @timestamp/`, ['@timestamp, ', '@timestamp | '] .map((text) => ({ text, @@ -1150,10 +1170,15 @@ describe('autocomplete', () => { })) .map(attachTriggerCommand), undefined, - [[{ name: '@timestamp', type: 'date' }]] + [ + [ + { name: '@timestamp', type: 'date' }, + { name: 'utc_stamp', type: 'date' }, + ], + ] ); testSuggestions( - 'FROM a | KEEP foo.bar/', + `FROM a | ${commandName} foo.bar/`, ['foo.bar, ', 'foo.bar | '] .map((text) => ({ text, @@ -1162,26 +1187,34 @@ describe('autocomplete', () => { })) .map(attachTriggerCommand), undefined, - [[{ name: 'foo.bar', type: 'double' }]] + [ + [ + { name: 'foo.bar', type: 'double' }, + { name: 'baz', type: 'date' }, + ], + ] ); describe('escaped field names', () => { // This isn't actually the behavior we want, but this test is here // to make sure no weird suggestions start cropping up in this case. - testSuggestions('FROM a | KEEP `foo.bar`/', ['foo.bar'], undefined, [ + testSuggestions(`FROM a | ${commandName} \`foo.bar\`/`, ['foo.bar'], undefined, [ [{ name: 'foo.bar', type: 'double' }], ]); // @todo re-enable these tests when we can use AST to support this case - testSuggestions.skip('FROM a | KEEP `foo.bar`/', ['foo.bar, ', 'foo.bar | '], undefined, [ - [{ name: 'foo.bar', type: 'double' }], - ]); testSuggestions.skip( - 'FROM a | KEEP `foo`.`bar`/', + `FROM a | ${commandName} \`foo.bar\`/`, ['foo.bar, ', 'foo.bar | '], undefined, [[{ name: 'foo.bar', type: 'double' }]] ); - testSuggestions.skip('FROM a | KEEP `any#Char$Field`/', [ + testSuggestions.skip( + `FROM a | ${commandName} \`foo\`.\`bar\`/`, + ['foo.bar, ', 'foo.bar | '], + undefined, + [[{ name: 'foo.bar', type: 'double' }]] + ); + testSuggestions.skip(`FROM a | ${commandName} \`any#Char$Field\`/`, [ '`any#Char$Field`, ', '`any#Char$Field` | ', ]); @@ -1189,12 +1222,28 @@ describe('autocomplete', () => { // Subsequent fields testSuggestions( - 'FROM a | KEEP doubleField, dateFiel/', + `FROM a | ${commandName} doubleField, dateFiel/`, getFieldNamesByType('any') .filter((s) => s !== 'doubleField') .map(attachTriggerCommand) ); - testSuggestions('FROM a | KEEP doubleField, dateField/', ['dateField, ', 'dateField | ']); + testSuggestions(`FROM a | ${commandName} doubleField, dateField/`, [ + 'dateField, ', + 'dateField | ', + ]); + + // out of fields + testSuggestions( + `FROM a | ${commandName} doubleField, dateField/`, + ['dateField | '], + undefined, + [ + [ + { name: 'doubleField', type: 'double' }, + { name: 'dateField', type: 'date' }, + ], + ] + ); }); }); }); diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts index 2433f5d496521..6f9fb66a8c715 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts @@ -627,7 +627,7 @@ async function getExpressionSuggestionsByType( literals: argDef.constantOnly, }, { - ignoreFields: isNewExpression + ignoreColumns: isNewExpression ? command.args.filter(isColumnItem).map(({ name }) => name) : [], } @@ -656,10 +656,15 @@ async function getExpressionSuggestionsByType( })); } - return [ - { ...pipeCompleteItem, text: ' | ' }, - { ...commaCompleteItem, text: ', ' }, - ].map((s) => ({ + const finalSuggestions = [{ ...pipeCompleteItem, text: ' | ' }]; + if (fieldSuggestions.length > 1) + // when we fix the editor marker, this should probably be checked against 0 instead of 1 + // this is because the last field in the AST is currently getting removed (because it contains + // the editor marker) so it is not included in the ignored list which is used to filter out + // existing fields above. + finalSuggestions.push({ ...commaCompleteItem, text: ', ' }); + + return finalSuggestions.map((s) => ({ ...s, filterText: fragment, text: fragment + s.text, @@ -1176,15 +1181,15 @@ async function getFieldsOrFunctionsSuggestions( }, { ignoreFn = [], - ignoreFields = [], + ignoreColumns = [], }: { ignoreFn?: string[]; - ignoreFields?: string[]; + ignoreColumns?: string[]; } = {} ): Promise { const filteredFieldsByType = pushItUpInTheList( (await (fields - ? getFieldsByType(types, ignoreFields, { + ? getFieldsByType(types, ignoreColumns, { advanceCursor: commandName === 'sort', openSuggestions: commandName === 'sort', }) @@ -1195,7 +1200,10 @@ async function getFieldsOrFunctionsSuggestions( const filteredVariablesByType: string[] = []; if (variables) { for (const variable of variables.values()) { - if (types.includes('any') || types.includes(variable[0].type)) { + if ( + (types.includes('any') || types.includes(variable[0].type)) && + !ignoreColumns.includes(variable[0].name) + ) { filteredVariablesByType.push(variable[0].name); } } @@ -1515,7 +1523,7 @@ async function getListArgsSuggestions( fields: true, variables: anyVariables, }, - { ignoreFields: [firstArg.name, ...otherArgs.map(({ name }) => name)] } + { ignoreColumns: [firstArg.name, ...otherArgs.map(({ name }) => name)] } )) ); } @@ -1875,18 +1883,16 @@ async function getOptionArgsSuggestions( * for a given fragment of text in a generic way. A good example is * a field name. * - * When typing a field name, there are three scenarios + * When typing a field name, there are 2 scenarios * - * 1. user hasn't begun typing + * 1. field name is incomplete (includes the empty string) * KEEP / - * - * 2. user is typing a partial field name * KEEP fie/ * - * 3. user has typed a complete field name + * 2. field name is complete * KEEP field/ * - * This function provides a framework for handling all three scenarios in a clean way. + * This function provides a framework for detecting and handling both scenarios in a clean way. * * @param innerText - the query text before the current cursor position * @param isFragmentComplete — return true if the fragment is complete diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 2eaeb64f8be5f..3572781c4b262 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -87,6 +87,9 @@ export const IGNORE_FILE_GLOBS = [ // Support for including http-client.env.json configurations '**/http-client.env.json', + + // updatecli configuration for driving the UBI/Ironbank image updates + 'updatecli-compose.yaml', ]; /** diff --git a/src/plugins/bfetch/server/ui_settings.ts b/src/plugins/bfetch/server/ui_settings.ts index aee4903d226c0..132dd19ef8b9c 100644 --- a/src/plugins/bfetch/server/ui_settings.ts +++ b/src/plugins/bfetch/server/ui_settings.ts @@ -18,7 +18,7 @@ export function getUiSettings(): Record> { name: i18n.translate('bfetch.disableBfetch', { defaultMessage: 'Disable request batching', }), - value: false, + value: true, description: i18n.translate('bfetch.disableBfetchDesc', { defaultMessage: 'Disables requests batching. This increases number of HTTP requests from Kibana, but allows to debug requests individually.', diff --git a/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts index 28819f7a5c54b..1719adebe7a49 100644 --- a/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts @@ -12,6 +12,7 @@ import { DataViewField } from '@kbn/data-views-plugin/common'; import { deepMockedFields, buildDataViewMock } from '@kbn/discover-utils/src/__mocks__'; import { allSuggestionsMock } from '../__mocks__/suggestions'; import { getLensVisMock } from '../__mocks__/lens_vis'; +import { convertDatatableColumnToDataViewFieldSpec } from '@kbn/data-view-utils'; import { UnifiedHistogramSuggestionType } from '../types'; describe('LensVisService suggestions', () => { @@ -198,6 +199,11 @@ describe('LensVisService suggestions', () => { }); test('should return histogramSuggestion if no suggestions returned by the api with the breakdown field if it is given', async () => { + const breakdown = convertDatatableColumnToDataViewFieldSpec({ + name: 'var0', + id: 'var0', + meta: { type: 'number' }, + }); const lensVis = await getLensVisMock({ filters: [], query: { esql: 'from the-data-view | limit 100' }, @@ -207,7 +213,7 @@ describe('LensVisService suggestions', () => { from: '2023-09-03T08:00:00.000Z', to: '2023-09-04T08:56:28.274Z', }, - breakdownField: { name: 'var0' } as DataViewField, + breakdownField: breakdown as DataViewField, columns: [ { id: 'var0', @@ -247,4 +253,54 @@ describe('LensVisService suggestions', () => { expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); }); + + test('should return histogramSuggestion if no suggestions returned by the api with a geo point breakdown field correctly', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | limit 100' }, + dataView: dataViewMock, + timeInterval: 'auto', + timeRange: { + from: '2023-09-03T08:00:00.000Z', + to: '2023-09-04T08:56:28.274Z', + }, + breakdownField: { name: 'coordinates' } as DataViewField, + columns: [ + { + id: 'coordinates', + name: 'coordinates', + meta: { + type: 'geo_point', + }, + }, + ], + isPlainRecord: true, + allSuggestions: [], + hasHistogramSuggestionForESQL: true, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe( + UnifiedHistogramSuggestionType.histogramForESQL + ); + expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined(); + expect(lensVis.currentSuggestionContext?.suggestion?.visualizationState).toHaveProperty( + 'layers', + [ + { + layerId: '662552df-2cdc-4539-bf3b-73b9f827252c', + seriesType: 'bar_stacked', + xAccessor: '@timestamp every 30 second', + accessors: ['results'], + layerType: 'data', + }, + ] + ); + + const histogramQuery = { + esql: `from the-data-view | limit 100 +| EVAL timestamp=DATE_TRUNC(30 minute, @timestamp) | stats results = count(*) by timestamp, \`coordinates\` | rename timestamp as \`@timestamp every 30 minute\``, + }; + + expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); + }); }); diff --git a/src/plugins/unified_histogram/public/services/lens_vis_service.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.ts index eccfd663b2557..25bb8be6f6242 100644 --- a/src/plugins/unified_histogram/public/services/lens_vis_service.ts +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.ts @@ -9,7 +9,11 @@ import { BehaviorSubject, distinctUntilChanged, map, Observable } from 'rxjs'; import { isEqual } from 'lodash'; -import { removeDropCommandsFromESQLQuery, appendToESQLQuery } from '@kbn/esql-utils'; +import { + removeDropCommandsFromESQLQuery, + appendToESQLQuery, + isESQLColumnSortable, +} from '@kbn/esql-utils'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import type { CountIndexPatternColumn, @@ -553,12 +557,17 @@ export class LensVisService { const queryInterval = interval ?? computeInterval(timeRange, this.services.data); const language = getAggregateQueryMode(query); const safeQuery = removeDropCommandsFromESQLQuery(query[language]); - const breakdown = breakdownColumn - ? `, \`${breakdownColumn.name}\` | sort \`${breakdownColumn.name}\` asc` - : ''; + const breakdown = breakdownColumn ? `, \`${breakdownColumn.name}\`` : ''; + + // sort by breakdown column if it's sortable + const sortBy = + breakdownColumn && isESQLColumnSortable(breakdownColumn) + ? ` | sort \`${breakdownColumn.name}\` asc` + : ''; + return appendToESQLQuery( safeQuery, - `| EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp${breakdown} | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\`` + `| EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp${breakdown}${sortBy} | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\`` ); }; diff --git a/updatecli-compose.yaml b/updatecli-compose.yaml new file mode 100644 index 0000000000000..8ad9bd6df8afb --- /dev/null +++ b/updatecli-compose.yaml @@ -0,0 +1,14 @@ +# Config file for `updatecli compose ...`. +# https://www.updatecli.io/docs/core/compose/ +policies: + - name: Handle ironbank bumps + policy: ghcr.io/elastic/oblt-updatecli-policies/ironbank/templates:0.3.0@sha256:b0c841d8fb294e6b58359462afbc83070dca375ac5dd0c5216c8926872a98bb1 + values: + - .github/updatecli/values.d/scm.yml + - .github/updatecli/values.d/ironbank.yml + + - name: Update Updatecli policies + policy: ghcr.io/updatecli/policies/autodiscovery/updatecli:0.4.0@sha256:254367f5b1454fd6032b88b314450cd3b6d5e8d5b6c953eb242a6464105eb869 + values: + - .github/updatecli/values.d/scm.yml + - .github/updatecli/values.d/updatecli-compose.yml \ No newline at end of file diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts index f6f27e15ee7a4..955f5a45a9743 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts @@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common'; import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest'; import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils'; @@ -39,6 +40,20 @@ interface FindingsAggs { count: estypes.AggregationsMultiBucketAggregateBase; } +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getFindingsQuery = ( { query, sort }: UseFindingsOptions, rulesStates: CspBenchmarkRulesStates, @@ -49,6 +64,7 @@ export const getFindingsQuery = ( return { index: CDR_MISCONFIGURATIONS_INDEX_PATTERN, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, aggs: getFindingsCountAggQuery(), ignore_unavailable: true, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index cc409fb95024d..6482d864347a1 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, + 'resource.id': { + type: 'keyword', + }, + 'resource.sub_type': { + type: 'keyword', + }, + 'resource.type': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.RULE_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.version': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a FindingsRootGroupingAggregation */ @@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: { + ...getRuntimeMappingsByGroupField(currentSelectedGroup), + 'result.evaluation': { + type: 'keyword', + }, + }, rootAggregations: [ { failedFindings: { diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx index 0d0ea9ba5a22f..0b9cf6978c258 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx @@ -23,6 +23,7 @@ import { } from '@kbn/cloud-security-posture-common'; import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture'; import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { VULNERABILITY_FIELDS } from '../../../common/constants'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script'; @@ -52,6 +53,25 @@ const getMultiFieldsSort = (sort: string[][]) => { }); }; +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = + field === VULNERABILITY_FIELDS.SCORE_BASE + ? 'double' + : field === '@timestamp' + ? 'date' + : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getVulnerabilitiesQuery = ( { query, sort }: VulnerabilitiesQuery, pageParam: number @@ -59,6 +79,7 @@ export const getVulnerabilitiesQuery = ( index: CDR_VULNERABILITIES_INDEX_PATTERN, ignore_unavailable: true, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, query: { ...query, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index 516cbed0c3975..3c52590f8fd80 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.RESOURCE_ID]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.CVE: + return { + [VULNERABILITY_GROUPING_OPTIONS.CVE]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.DESCRIPTION]: { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation */ @@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup), }); const { data, isFetching } = useGroupedVulnerabilities({ diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts index 780cd539305b3..e517d622e71c5 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts @@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) = type: 'string', order: direction, script: { - source: `doc["${field}"].value.toLowerCase()`, + source: ` + if (doc.containsKey('${field}') && !doc['${field}'].empty) { + return doc['${field}'].value.toLowerCase(); + } else { + return ""; + } + `, lang: 'painless', }, }, diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts index f6c08e2caddc0..473e64c6b03d9 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts @@ -10,48 +10,29 @@ import { UsageMetricsRequestSchema } from './usage_metrics'; describe('usage_metrics schemas', () => { it('should accept valid request query', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], - }) - ).not.toThrow(); - }); - - it('should accept a single `metricTypes` in request query', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: new Date().toISOString(), - metricTypes: 'ingest_rate', + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], }) ).not.toThrow(); }); it('should accept multiple `metricTypes` in request query', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['ingest_rate', 'storage_retained', 'index_rate'], - }) - ).not.toThrow(); - }); - - it('should accept a single string as `dataStreams` in request query', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: new Date().toISOString(), - metricTypes: 'storage_retained', - dataStreams: 'data_stream_1', + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], }) ).not.toThrow(); }); it('should accept `dataStream` list', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], @@ -62,74 +43,76 @@ describe('usage_metrics schemas', () => { it('should error if `dataStream` list is empty', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], dataStreams: [], }) - ).toThrowError('expected value of type [string] but got [Array]'); + ).toThrowError('[dataStreams]: array size is [0], but cannot be smaller than [1]'); }); - it('should error if `dataStream` is given an empty string', () => { + it('should error if `dataStream` is given type not array', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], dataStreams: ' ', }) - ).toThrow('[dataStreams] must have at least one value'); + ).toThrow('[dataStreams]: could not parse array value from json input'); }); it('should error if `dataStream` is given an empty item in the list', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], dataStreams: ['ds_1', ' '], }) - ).toThrow('[dataStreams] list can not contain empty values'); + ).toThrow('[dataStreams]: [dataStreams] list cannot contain empty values'); }); it('should error if `metricTypes` is empty string', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: ' ', }) ).toThrow(); }); - it('should error if `metricTypes` is empty item', () => { + it('should error if `metricTypes` contains an empty item', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), - metricTypes: [' ', 'storage_retained'], + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], + metricTypes: [' ', 'storage_retained'], // First item is invalid }) - ).toThrow('[metricTypes] list can not contain empty values'); + ).toThrowError(/list cannot contain empty values/); }); - it('should error if `metricTypes` is not a valid value', () => { + it('should error if `metricTypes` is not a valid type', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: 'foo', }) - ).toThrow( - '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' - ); + ).toThrow('[metricTypes]: could not parse array value from json input'); }); it('should error if `metricTypes` is not a valid list', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: new Date().toISOString(), + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: ['storage_retained', 'foo'], }) ).toThrow( @@ -139,9 +122,10 @@ describe('usage_metrics schemas', () => { it('should error if `from` is not a valid input', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: 1010, to: new Date().toISOString(), + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: ['storage_retained', 'foo'], }) ).toThrow('[from]: expected value of type [string] but got [number]'); @@ -149,9 +133,10 @@ describe('usage_metrics schemas', () => { it('should error if `to` is not a valid input', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: 1010, + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: ['storage_retained', 'foo'], }) ).toThrow('[to]: expected value of type [string] but got [number]'); @@ -159,9 +144,10 @@ describe('usage_metrics schemas', () => { it('should error if `from` is empty string', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: ' ', to: new Date().toISOString(), + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: ['storage_retained', 'foo'], }) ).toThrow('[from]: Date ISO string must not be empty'); @@ -169,9 +155,10 @@ describe('usage_metrics schemas', () => { it('should error if `to` is empty string', () => { expect(() => - UsageMetricsRequestSchema.query.validate({ + UsageMetricsRequestSchema.validate({ from: new Date().toISOString(), to: ' ', + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], metricTypes: ['storage_retained', 'foo'], }) ).toThrow('[to]: Date ISO string must not be empty'); diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index f2bbdb616fc79..3dceeadc198b0 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -37,51 +37,31 @@ const metricTypesSchema = schema.oneOf( // @ts-expect-error TS2769: No overload matches this call METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType)) // Create a oneOf schema for the keys ); -export const UsageMetricsRequestSchema = { - query: schema.object({ - from: DateSchema, - to: DateSchema, - metricTypes: schema.oneOf([ - schema.arrayOf(schema.string(), { - minSize: 1, - validate: (values) => { - if (values.map((v) => v.trim()).some((v) => !v.length)) { - return '[metricTypes] list can not contain empty values'; - } else if (values.map((v) => v.trim()).some((v) => !isValidMetricType(v))) { - return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; - } - }, - }), - schema.string({ - validate: (v) => { - if (!v.trim().length) { - return '[metricTypes] must have at least one value'; - } else if (!isValidMetricType(v)) { - return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; - } - }, - }), - ]), - dataStreams: schema.maybe( - schema.oneOf([ - schema.arrayOf(schema.string(), { - minSize: 1, - validate: (values) => { - if (values.map((v) => v.trim()).some((v) => !v.length)) { - return '[dataStreams] list can not contain empty values'; - } - }, - }), - schema.string({ - validate: (v) => - v.trim().length ? undefined : '[dataStreams] must have at least one value', - }), - ]) - ), +export const UsageMetricsRequestSchema = schema.object({ + from: DateSchema, + to: DateSchema, + metricTypes: schema.arrayOf(schema.string(), { + minSize: 1, + validate: (values) => { + const trimmedValues = values.map((v) => v.trim()); + if (trimmedValues.some((v) => !v.length)) { + return '[metricTypes] list cannot contain empty values'; + } else if (trimmedValues.some((v) => !isValidMetricType(v))) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, }), -}; + dataStreams: schema.arrayOf(schema.string(), { + minSize: 1, + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[dataStreams] list cannot contain empty values'; + } + }, + }), +}); -export type UsageMetricsRequestSchemaQueryParams = TypeOf; +export type UsageMetricsRequestSchemaQueryParams = TypeOf; export const UsageMetricsResponseSchema = { body: () => @@ -92,11 +72,40 @@ export const UsageMetricsResponseSchema = { schema.object({ name: schema.string(), data: schema.arrayOf( - schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 }) // Each data point is an array of 2 numbers + schema.object({ + x: schema.number(), + y: schema.number(), + }) ), }) ) ), }), }; -export type UsageMetricsResponseSchemaBody = TypeOf; +export type UsageMetricsResponseSchemaBody = Omit< + TypeOf, + 'metrics' +> & { + metrics: Partial>; +}; +export type MetricSeries = TypeOf< + typeof UsageMetricsResponseSchema.body +>['metrics'][MetricTypes][number]; + +export const UsageMetricsAutoOpsResponseSchema = { + body: () => + schema.object({ + metrics: schema.recordOf( + metricTypesSchema, + schema.arrayOf( + schema.object({ + name: schema.string(), + data: schema.arrayOf(schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 })), + }) + ) + ), + }), +}; +export type UsageMetricsAutoOpsResponseSchemaBody = TypeOf< + typeof UsageMetricsAutoOpsResponseSchema.body +>; diff --git a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx index c7937ae149de9..1ba3f0fe3f454 100644 --- a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx +++ b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx @@ -19,8 +19,7 @@ import { } from '@elastic/charts'; import { i18n } from '@kbn/i18n'; import { LegendAction } from './legend_action'; -import { MetricTypes } from '../../../common/rest_types'; -import { MetricSeries } from '../types'; +import { MetricTypes, MetricSeries } from '../../../common/rest_types'; // TODO: Remove this when we have a title for each metric type type ChartKey = Extract; @@ -50,7 +49,7 @@ export const ChartPanel: React.FC = ({ }) => { const theme = useEuiTheme(); - const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d[0])); + const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d.x)); const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)]; @@ -72,6 +71,7 @@ export const ChartPanel: React.FC = ({ }, [idx, popoverOpen, togglePopover] ); + return ( @@ -94,9 +94,9 @@ export const ChartPanel: React.FC = ({ data={stream.data} xScaleType={ScaleType.Time} yScaleType={ScaleType.Linear} - xAccessor={0} // x is the first element in the tuple - yAccessors={[1]} // y is the second element in the tuple - stackAccessors={[0]} + xAccessor="x" + yAccessors={['y']} + stackAccessors={['x']} /> ))} @@ -118,6 +118,7 @@ export const ChartPanel: React.FC = ({ ); }; + const formatBytes = (bytes: number) => { return numeral(bytes).format('0.0 b'); }; diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 6549f7e03830a..8d04324fb2246 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -6,11 +6,11 @@ */ import React, { useCallback, useState } from 'react'; import { EuiFlexGroup } from '@elastic/eui'; -import { MetricsResponse } from '../types'; import { MetricTypes } from '../../../common/rest_types'; import { ChartPanel } from './chart_panel'; +import { UsageMetricsResponseSchemaBody } from '../../../common/rest_types'; interface ChartsProps { - data: MetricsResponse; + data: UsageMetricsResponseSchemaBody; } export const Charts: React.FC = ({ data }) => { diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index c32f86d68b5bf..bea9f2b511a77 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -26,7 +26,6 @@ import { PLUGIN_NAME } from '../../common'; import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; import { DEFAULT_DATE_RANGE_OPTIONS, useDateRangePicker } from './hooks/use_date_picker'; import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params'; -import { MetricsResponse } from './types'; export const DataUsage = () => { const { @@ -42,37 +41,37 @@ export const DataUsage = () => { setUrlDateRangeFilter, } = useDataUsageMetricsUrlParams(); - const [queryParams, setQueryParams] = useState({ + const [metricsFilters, setMetricsFilters] = useState({ metricTypes: ['storage_retained', 'ingest_rate'], - dataStreams: [], + // TODO: Replace with data streams from /data_streams api + dataStreams: [ + '.alerts-ml.anomaly-detection-health.alerts-default', + '.alerts-stack.alerts-default', + ], from: DEFAULT_DATE_RANGE_OPTIONS.startDate, to: DEFAULT_DATE_RANGE_OPTIONS.endDate, }); useEffect(() => { if (!metricTypesFromUrl) { - setUrlMetricTypesFilter( - typeof queryParams.metricTypes !== 'string' - ? queryParams.metricTypes.join(',') - : queryParams.metricTypes - ); + setUrlMetricTypesFilter(metricsFilters.metricTypes.join(',')); } if (!startDateFromUrl || !endDateFromUrl) { - setUrlDateRangeFilter({ startDate: queryParams.from, endDate: queryParams.to }); + setUrlDateRangeFilter({ startDate: metricsFilters.from, endDate: metricsFilters.to }); } }, [ endDateFromUrl, metricTypesFromUrl, - queryParams.from, - queryParams.metricTypes, - queryParams.to, + metricsFilters.from, + metricsFilters.metricTypes, + metricsFilters.to, setUrlDateRangeFilter, setUrlMetricTypesFilter, startDateFromUrl, ]); useEffect(() => { - setQueryParams((prevState) => ({ + setMetricsFilters((prevState) => ({ ...prevState, metricTypes: metricTypesFromUrl?.length ? metricTypesFromUrl : prevState.metricTypes, dataStreams: dataStreamsFromUrl?.length ? dataStreamsFromUrl : prevState.dataStreams, @@ -89,7 +88,7 @@ export const DataUsage = () => { refetch: refetchDataUsageMetrics, } = useGetDataUsageMetrics( { - ...queryParams, + ...metricsFilters, from: dateRangePickerState.startDate, to: dateRangePickerState.endDate, }, @@ -140,7 +139,7 @@ export const DataUsage = () => { - {isFetched && data ? : } + {isFetched && data ? : } ); diff --git a/x-pack/plugins/data_usage/public/app/types.ts b/x-pack/plugins/data_usage/public/app/types.ts deleted file mode 100644 index 13f53bc2ea6dd..0000000000000 --- a/x-pack/plugins/data_usage/public/app/types.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { MetricTypes } from '../../common/rest_types'; - -export type DataPoint = [number, number]; // [timestamp, value] - -export interface MetricSeries { - name: string; // Name of the data stream - data: DataPoint[]; // Array of data points in tuple format [timestamp, value] -} -// Use MetricTypes dynamically as keys for the Metrics interface -export type Metrics = Partial>; - -export interface MetricsResponse { - metrics: Metrics; -} -export interface MetricsResponse { - metrics: Metrics; -} diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts index 6b9860e997c12..3d648eb183f07 100644 --- a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -21,24 +21,24 @@ interface ErrorType { } export const useGetDataUsageMetrics = ( - query: UsageMetricsRequestSchemaQueryParams, + body: UsageMetricsRequestSchemaQueryParams, options: UseQueryOptions> = {} ): UseQueryResult> => { const http = useKibanaContextForPlugin().services.http; return useQuery>({ - queryKey: ['get-data-usage-metrics', query], + queryKey: ['get-data-usage-metrics', body], ...options, keepPreviousData: true, queryFn: async () => { - return http.get(DATA_USAGE_METRICS_API_ROUTE, { + return http.post(DATA_USAGE_METRICS_API_ROUTE, { version: '1', - query: { - from: query.from, - to: query.to, - metricTypes: query.metricTypes, - dataStreams: query.dataStreams, - }, + body: JSON.stringify({ + from: body.from, + to: body.to, + metricTypes: body.metricTypes, + dataStreams: body.dataStreams, + }), }); }, }); diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts index 5bf3008ef668a..0013102f697fb 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -17,7 +17,7 @@ export const registerUsageMetricsRoute = ( ) => { if (dataUsageContext.serverConfig.enabled) { router.versioned - .get({ + .post({ access: 'internal', path: DATA_USAGE_METRICS_API_ROUTE, }) @@ -25,7 +25,9 @@ export const registerUsageMetricsRoute = ( { version: '1', validate: { - request: UsageMetricsRequestSchema, + request: { + body: UsageMetricsRequestSchema, + }, response: { 200: UsageMetricsResponseSchema, }, diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 6f992c9fb2a38..09e9f88721c63 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -9,8 +9,10 @@ import { RequestHandler } from '@kbn/core/server'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; import { MetricTypes, + UsageMetricsAutoOpsResponseSchema, + UsageMetricsAutoOpsResponseSchemaBody, UsageMetricsRequestSchemaQueryParams, - UsageMetricsResponseSchema, + UsageMetricsResponseSchemaBody, } from '../../../common/rest_types'; import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; @@ -34,45 +36,26 @@ export const getUsageMetricsHandler = ( const core = await context.core; const esClient = core.elasticsearch.client.asCurrentUser; - // @ts-ignore - const { from, to, metricTypes, dataStreams: dsNames, size } = request.query; + const { from, to, metricTypes, dataStreams: requestDsNames } = request.query; logger.debug(`Retrieving usage metrics`); const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse = await esClient.indices.getDataStream({ - name: '*', + name: requestDsNames, expand_wildcards: 'all', }); - const hasDataStreams = dataStreamsResponse.length > 0; - let userDsNames: string[] = []; - - if (dsNames?.length) { - userDsNames = typeof dsNames === 'string' ? [dsNames] : dsNames; - } else if (!userDsNames.length && hasDataStreams) { - userDsNames = dataStreamsResponse.map((ds) => ds.name); - } - - // If no data streams are found, return an empty response - if (!userDsNames.length) { - return response.ok({ - body: { - metrics: {}, - }, - }); - } - const metrics = await fetchMetricsFromAutoOps({ from, to, metricTypes: formatStringParams(metricTypes) as MetricTypes[], - dataStreams: formatStringParams(userDsNames), + dataStreams: formatStringParams(dataStreamsResponse.map((ds) => ds.name)), }); + const processedMetrics = transformMetricsData(metrics); + return response.ok({ - body: { - metrics, - }, + body: processedMetrics, }); } catch (error) { logger.error(`Error retrieving usage metrics: ${error.message}`); @@ -94,7 +77,7 @@ const fetchMetricsFromAutoOps = async ({ }) => { // TODO: fetch data from autoOps using userDsNames /* - const response = await axios.post('https://api.auto-ops.{region}.{csp}.cloud.elastic.co/monitoring/serverless/v1/projects/{project_id}/metrics', { + const response = await axios.post({AUTOOPS_URL}, { from: Date.parse(from), to: Date.parse(to), metric_types: metricTypes, @@ -231,7 +214,25 @@ const fetchMetricsFromAutoOps = async ({ }, }; // Make sure data is what we expect - const validatedData = UsageMetricsResponseSchema.body().validate(mockData); + const validatedData = UsageMetricsAutoOpsResponseSchema.body().validate(mockData); - return validatedData.metrics; + return validatedData; }; +function transformMetricsData( + data: UsageMetricsAutoOpsResponseSchemaBody +): UsageMetricsResponseSchemaBody { + return { + metrics: Object.fromEntries( + Object.entries(data.metrics).map(([metricType, series]) => [ + metricType, + series.map((metricSeries) => ({ + name: metricSeries.name, + data: (metricSeries.data as Array<[number, number]>).map(([timestamp, value]) => ({ + x: timestamp, + y: value, + })), + })), + ]) + ), + }; +} diff --git a/x-pack/plugins/index_lifecycle_management/server/plugin.ts b/x-pack/plugins/index_lifecycle_management/server/plugin.ts index 0d88acbaaa4ff..a5002cd36da44 100644 --- a/x-pack/plugins/index_lifecycle_management/server/plugin.ts +++ b/x-pack/plugins/index_lifecycle_management/server/plugin.ts @@ -27,7 +27,6 @@ const indexLifecycleDataEnricher = async ( const { indices: ilmIndicesData } = await client.asCurrentUser.ilm.explainLifecycle({ index: '*,.*', - only_managed: true, }); return indicesList.map((index: Index) => { return { diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 1ae20af759611..1e5ffee50afc7 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -138,11 +138,6 @@ export const allowedExperimentalValues = Object.freeze({ */ esqlRulesDisabled: false, - /** - * enables logging requests during rule preview - */ - loggingRequestsEnabled: false, - /** * Enables Protection Updates tab in the Endpoint Policy Details page */ diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx index 4ebb460177476..25d5b90d5408a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.test.tsx @@ -23,7 +23,6 @@ import { stepDefineDefaultValue, } from '../../../../detections/pages/detection_engine/rules/utils'; import { usePreviewInvocationCount } from './use_preview_invocation_count'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; jest.mock('../../../../common/lib/kibana'); jest.mock('./use_preview_route'); @@ -40,7 +39,6 @@ jest.mock('../../../../common/hooks/use_experimental_features', () => ({ useIsExperimentalFeatureEnabled: jest.fn(), })); -const useIsExperimentalFeatureEnabledMock = useIsExperimentalFeatureEnabled as jest.Mock; // rule types that do not support logged requests const doNotSupportLoggedRequests: Type[] = [ 'threshold', @@ -114,8 +112,6 @@ describe('PreviewQuery', () => { }); (usePreviewInvocationCount as jest.Mock).mockReturnValue({ invocationCount: 500 }); - - useIsExperimentalFeatureEnabledMock.mockReturnValue(true); }); afterEach(() => { @@ -172,23 +168,6 @@ describe('PreviewQuery', () => { }); }); - supportLoggedRequests.forEach((ruleType) => { - test(`does not render "Show Elasticsearch requests" for ${ruleType} rule type when feature is disabled`, () => { - useIsExperimentalFeatureEnabledMock.mockReturnValue(false); - - render( - - - - ); - - expect(screen.queryByTestId('show-elasticsearch-requests')).toBeNull(); - }); - }); - doNotSupportLoggedRequests.forEach((ruleType) => { test(`does not render "Show Elasticsearch requests" for ${ruleType} rule type`, () => { render( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx index 2a86600d94e7a..f941cad91d3a4 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/index.tsx @@ -40,7 +40,6 @@ import type { TimeframePreviewOptions, } from '../../../../detections/pages/detection_engine/rules/types'; import { usePreviewInvocationCount } from './use_preview_invocation_count'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; export const REASONABLE_INVOCATION_COUNT = 200; @@ -90,8 +89,6 @@ const RulePreviewComponent: React.FC = ({ const { indexPattern, ruleType } = defineRuleData; const { spaces } = useKibana().services; - const isLoggingRequestsFeatureEnabled = useIsExperimentalFeatureEnabled('loggingRequestsEnabled'); - const [spaceId, setSpaceId] = useState(''); useEffect(() => { if (spaces) { @@ -282,8 +279,7 @@ const RulePreviewComponent: React.FC = ({ - {isLoggingRequestsFeatureEnabled && - RULE_TYPES_SUPPORTING_LOGGED_REQUESTS.includes(ruleType) ? ( + {RULE_TYPES_SUPPORTING_LOGGED_REQUESTS.includes(ruleType) ? ( diff --git a/x-pack/test/api_integration/apis/management/index_management/data_enrichers/ilm.ts b/x-pack/test/api_integration/apis/management/index_management/data_enrichers/ilm.ts index 234a1518a9c59..3ae9b554bf3ee 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_enrichers/ilm.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_enrichers/ilm.ts @@ -55,16 +55,17 @@ export default function ({ getService }: FtrProviderContext) { const testAlias = 'test_alias'; const testIlmPolicy = 'test_policy'; describe('GET indices with data enrichers', () => { - before(async () => { + beforeEach(async () => { await createIndex(testIndex); - await createIlmPolicy('test_policy'); - await addPolicyToIndex(testIlmPolicy, testIndex, testAlias); }); - after(async () => { + afterEach(async () => { await esDeleteAllIndices([testIndex]); }); it(`ILM data is fetched by the ILM data enricher`, async () => { + await createIlmPolicy('test_policy'); + await addPolicyToIndex(testIlmPolicy, testIndex, testAlias); + const { body: indices } = await supertest .get(`${API_BASE_PATH}/indices`) .set('kbn-xsrf', 'xxx') @@ -75,5 +76,18 @@ export default function ({ getService }: FtrProviderContext) { const { ilm } = index; expect(ilm.policy).to.eql(testIlmPolicy); }); + + it(`ILM data is not empty even if the index unmanaged`, async () => { + const { body: indices } = await supertest + .get(`${API_BASE_PATH}/indices`) + .set('kbn-xsrf', 'xxx') + .expect(200); + + const index = indices.find((item: Index) => item.name === testIndex); + + const { ilm } = index; + expect(ilm.index).to.eql(testIndex); + expect(ilm.managed).to.eql(false); + }); }); } diff --git a/x-pack/test/api_integration/apis/ml/anomaly_detectors/forecast_with_spaces.ts b/x-pack/test/api_integration/apis/ml/anomaly_detectors/forecast_with_spaces.ts index a330edd9a41d7..32e82c67e348d 100644 --- a/x-pack/test/api_integration/apis/ml/anomaly_detectors/forecast_with_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/anomaly_detectors/forecast_with_spaces.ts @@ -58,7 +58,8 @@ export default ({ getService }: FtrProviderContext) => { return body; } - describe('POST anomaly_detectors _forecast with spaces', function () { + // Failing see: https://github.com/elastic/kibana/issues/195602 + describe.skip('POST anomaly_detectors _forecast with spaces', function () { let forecastId: string; before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); diff --git a/x-pack/test/functional_execution_context/tests/browser.ts b/x-pack/test/functional_execution_context/tests/browser.ts index e1d7ba6a3b965..c7228528ee756 100644 --- a/x-pack/test/functional_execution_context/tests/browser.ts +++ b/x-pack/test/functional_execution_context/tests/browser.ts @@ -85,14 +85,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { type: 'application', name: 'discover', url: '/app/discover', - child: { - name: 'discover', - url: '/app/discover', - type: 'application', - page: 'app', - id: 'new', - description: 'fetch documents', - }, + page: 'app', + id: 'new', + description: 'fetch documents', }), }); }); @@ -105,20 +100,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { type: 'application', name: 'discover', url: '/app/discover', + page: 'app', + id: 'new', + description: 'fetch chart data and total hits', child: { - name: 'discover', - url: '/app/discover', - type: 'application', - page: 'app', - id: 'new', - description: 'fetch chart data and total hits', - child: { - type: 'lens', - name: 'lnsXY', - id: 'unifiedHistogramLensComponent', - description: 'Edit visualization', - url: '/app/lens#/edit_by_value', - }, + type: 'lens', + name: 'lnsXY', + id: 'unifiedHistogramLensComponent', + description: 'Edit visualization', + url: '/app/lens#/edit_by_value', }, }), }); @@ -185,9 +175,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', - predicate: checkHttpRequestId( - 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsXY:086ac2e9-dd16-4b45-92b8-1e43ff7e3f65' - ), + predicate: checkHttpRequestId('lens:lnsXY:086ac2e9-dd16-4b45-92b8-1e43ff7e3f65'), }); }); @@ -195,23 +183,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ - type: 'application', + type: 'dashboard', name: 'dashboards', url: '/app/dashboards', + page: 'app', + id: '7adfa750-4c81-11e8-b3d7-01146121b73d', + description: '[Flights] Global Flight Dashboard', child: { - name: 'dashboards', - url: '/app/dashboards', - type: 'dashboard', - page: 'app', - id: '7adfa750-4c81-11e8-b3d7-01146121b73d', - description: '[Flights] Global Flight Dashboard', - child: { - type: 'lens', - name: 'lnsXY', - id: '086ac2e9-dd16-4b45-92b8-1e43ff7e3f65', - description: '[Flights] Flight count', - url: '/app/lens#/edit_by_value', - }, + type: 'lens', + name: 'lnsXY', + id: '086ac2e9-dd16-4b45-92b8-1e43ff7e3f65', + description: '[Flights] Flight count', + url: '/app/lens#/edit_by_value', }, }), }); @@ -222,9 +205,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', - predicate: checkHttpRequestId( - 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsMetric:b766e3b8-4544-46ed-99e6-9ecc4847e2a2' - ), + predicate: checkHttpRequestId('lens:lnsMetric:b766e3b8-4544-46ed-99e6-9ecc4847e2a2'), }); }); @@ -232,23 +213,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ + type: 'dashboard', name: 'dashboards', url: '/app/dashboards', - type: 'application', + page: 'app', + id: '7adfa750-4c81-11e8-b3d7-01146121b73d', + description: '[Flights] Global Flight Dashboard', child: { - name: 'dashboards', - url: '/app/dashboards', - type: 'dashboard', - page: 'app', - id: '7adfa750-4c81-11e8-b3d7-01146121b73d', - description: '[Flights] Global Flight Dashboard', - child: { - type: 'lens', - name: 'lnsMetric', - id: 'b766e3b8-4544-46ed-99e6-9ecc4847e2a2', - description: '', - url: '/app/lens#/edit_by_value', - }, + type: 'lens', + name: 'lnsMetric', + id: 'b766e3b8-4544-46ed-99e6-9ecc4847e2a2', + description: '', + url: '/app/lens#/edit_by_value', }, }), }); @@ -260,7 +236,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', predicate: checkHttpRequestId( - 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsDatatable:fb86b32f-fb7a-45cf-9511-f366fef51bbd' + 'lens:lnsDatatable:fb86b32f-fb7a-45cf-9511-f366fef51bbd' ), }); }); @@ -269,23 +245,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ + type: 'dashboard', name: 'dashboards', url: '/app/dashboards', - type: 'application', + page: 'app', + id: '7adfa750-4c81-11e8-b3d7-01146121b73d', + description: '[Flights] Global Flight Dashboard', child: { - name: 'dashboards', - url: '/app/dashboards', - type: 'dashboard', - page: 'app', - id: '7adfa750-4c81-11e8-b3d7-01146121b73d', - description: '[Flights] Global Flight Dashboard', - child: { - type: 'lens', - name: 'lnsDatatable', - id: 'fb86b32f-fb7a-45cf-9511-f366fef51bbd', - description: 'Cities by delay, cancellation', - url: '/app/lens#/edit_by_value', - }, + type: 'lens', + name: 'lnsDatatable', + id: 'fb86b32f-fb7a-45cf-9511-f366fef51bbd', + description: 'Cities by delay, cancellation', + url: '/app/lens#/edit_by_value', }, }), }); @@ -296,9 +267,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', - predicate: checkHttpRequestId( - 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;lens:lnsPie:5d53db36-2d5a-4adc-af7b-cec4c1a294e0' - ), + predicate: checkHttpRequestId('lens:lnsPie:5d53db36-2d5a-4adc-af7b-cec4c1a294e0'), }); }); @@ -306,23 +275,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ + type: 'dashboard', name: 'dashboards', url: '/app/dashboards', - type: 'application', + page: 'app', + id: '7adfa750-4c81-11e8-b3d7-01146121b73d', + description: '[Flights] Global Flight Dashboard', child: { - name: 'dashboards', - url: '/app/dashboards', - type: 'dashboard', - page: 'app', - id: '7adfa750-4c81-11e8-b3d7-01146121b73d', - description: '[Flights] Global Flight Dashboard', - child: { - type: 'lens', - name: 'lnsPie', - id: '5d53db36-2d5a-4adc-af7b-cec4c1a294e0', - description: '[Flights] Delay Type', - url: '/app/lens#/edit_by_value', - }, + type: 'lens', + name: 'lnsPie', + id: '5d53db36-2d5a-4adc-af7b-cec4c1a294e0', + description: '[Flights] Delay Type', + url: '/app/lens#/edit_by_value', }, }), }); @@ -334,9 +298,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('propagates to Elasticsearch via "x-opaque-id" header', async () => { await logContains({ description: 'execution context propagates to Elasticsearch via "x-opaque-id" header', - predicate: checkHttpRequestId( - 'dashboard:dashboards:7adfa750-4c81-11e8-b3d7-01146121b73d;search:discover:571aaf70-4c88-11e8-b3d7-01146121b73d' - ), + predicate: checkHttpRequestId('search:discover:571aaf70-4c88-11e8-b3d7-01146121b73d'), }); }); @@ -344,23 +306,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await logContains({ description: 'execution context propagates to Kibana logs', predicate: checkExecutionContextEntry({ - type: 'application', + type: 'dashboard', name: 'dashboards', url: '/app/dashboards', + page: 'app', + id: '7adfa750-4c81-11e8-b3d7-01146121b73d', + description: '[Flights] Global Flight Dashboard', child: { - type: 'dashboard', - name: 'dashboards', - url: '/app/dashboards', - page: 'app', - id: '7adfa750-4c81-11e8-b3d7-01146121b73d', - description: '[Flights] Global Flight Dashboard', - child: { - type: 'search', - name: 'discover', - id: '571aaf70-4c88-11e8-b3d7-01146121b73d', - description: '[Flights] Flight Log', - url: '/app/discover#/view/571aaf70-4c88-11e8-b3d7-01146121b73d', - }, + type: 'search', + name: 'discover', + id: '571aaf70-4c88-11e8-b3d7-01146121b73d', + description: '[Flights] Flight Log', + url: '/app/discover#/view/571aaf70-4c88-11e8-b3d7-01146121b73d', }, }), }); diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index 3ab6d5059fd07..a0d2ee79a7b46 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -82,7 +82,6 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', - 'loggingRequestsEnabled', 'riskScoringPersistence', 'riskScoringRoutesEnabled', ])}`, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts index ce949d5cc23fc..137ee1f67b9b3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -17,6 +17,5 @@ export default createTestConfig({ 'testing_ignored.constant', '/testing_regex*/', ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" - `--xpack.securitySolution.enableExperimental=${JSON.stringify(['loggingRequestsEnabled'])}`, ], }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index aff2ccc6bccb3..9077873274fa5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -1190,8 +1190,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - // skipped on MKI since feature flags are not supported there - describe('@skipInServerlessMKI preview logged requests', () => { + describe('preview logged requests', () => { it('should not return requests property when not enabled', async () => { const { logs } = await previewRule({ supertest, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts index 166a62b9b08ad..ee976de14186d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts @@ -1409,8 +1409,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - // skipped on MKI since feature flags are not supported there - describe('@skipInServerlessMKI preview logged requests', () => { + describe('preview logged requests', () => { let rule: EsqlRuleCreateProps; let id: string; beforeEach(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts index bd3493b82d348..19a9bb85326fa 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/init_and_status_apis.ts @@ -26,8 +26,7 @@ export default ({ getService }: FtrProviderContext) => { const riskEngineRoutes = riskEngineRouteHelpersFactory(supertest); const log = getService('log'); - // Failing: See https://github.com/elastic/kibana/issues/191637 - describe.skip('@ess @serverless @serverlessQA init_and_status_apis', () => { + describe('@ess @serverless @serverlessQA init_and_status_apis', () => { before(async () => { await riskEngineRoutes.cleanUp(); }); @@ -298,8 +297,8 @@ export default ({ getService }: FtrProviderContext) => { firstResponse?.saved_objects?.[0]?.id ); }); - - describe('remove legacy risk score transform', function () { + // Failing: See https://github.com/elastic/kibana/issues/191637 + describe.skip('remove legacy risk score transform', function () { this.tags('skipFIPS'); it('should remove legacy risk score transform if it exists', async () => { await installLegacyRiskScore({ supertest }); diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 05bc2e381527a..f02968945087d 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -44,7 +44,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // See https://github.com/elastic/kibana/pull/125396 for details '--xpack.alerting.rules.minimumScheduleInterval.value=1s', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', - `--xpack.securitySolution.enableExperimental=${JSON.stringify(['loggingRequestsEnabled'])}`, // mock cloud to enable the guided onboarding tour in e2e tests '--xpack.cloud.id=test', `--home.disableWelcomeScreen=true`, diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts index 0045a79ff4394..64423a921e595 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts @@ -67,7 +67,8 @@ const workaroundForResizeObserver = () => } }); -describe( +// Failing: See https://github.com/elastic/kibana/issues/184558 +describe.skip( 'Detection ES|QL rules, creation', { tags: ['@ess', '@serverless', '@skipInServerlessMKI'], diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts index 9fa45987407f0..34f301602b692 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts @@ -55,7 +55,9 @@ const expectedValidEsqlQuery = 'from auditbeat* | stats _count=count(event.category) by event.category'; // Skipping in MKI due to flake -describe( +// Failing: See https://github.com/elastic/kibana/issues/184557 +// Failing: See https://github.com/elastic/kibana/issues/184556 +describe.skip( 'Detection ES|QL rules, edit', { tags: ['@ess', '@serverless', '@skipInServerlessMKI'], diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts index c2e41c9d4680c..268968c76ecc0 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/preview.cy.ts @@ -33,11 +33,6 @@ describe( 'Detection rules, preview', { tags: ['@ess', '@serverless'], - env: { - kbnServerArgs: [ - `--xpack.securitySolution.enableExperimental=${JSON.stringify(['loggingRequestsEnabled'])}`, - ], - }, }, () => { beforeEach(() => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/backfill_group.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/backfill_group.cy.ts index f747e6be43e5a..6466c20dfde21 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/backfill_group.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_details/backfill_group.cy.ts @@ -34,7 +34,7 @@ import { describe( 'Backfill groups', { - tags: ['@ess', '@serverless'], + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], }, function () { before(() => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/vulnerabilities_contextual_flyout.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/vulnerabilities_contextual_flyout.cy.ts index 591d458af56c1..fb83df1c79141 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/vulnerabilities_contextual_flyout.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/vulnerabilities_contextual_flyout.cy.ts @@ -138,7 +138,8 @@ const deleteDataStream = () => { }); }; -describe('Alert Host details expandable flyout', { tags: ['@ess', '@serverless'] }, () => { +// skipping because failure on MKI environment (https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-investigations/builds/1390#01927579-caed-41bc-9440-3cf29629a263) +describe.skip('Alert Host details expandable flyout', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { deleteAlertsAndRules(); login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts index ebfc5d4e9a0cb..f485ead495949 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts @@ -67,7 +67,8 @@ const URL = '/app/security/threat_intelligence/indicators'; const URL_WITH_CONTRADICTORY_FILTERS = '/app/security/threat_intelligence/indicators?indicators=(filterQuery:(language:kuery,query:%27%27),filters:!((%27$state%27:(store:appState),meta:(alias:!n,disabled:!f,index:%27%27,key:threat.indicator.type,negate:!f,params:(query:file),type:phrase),query:(match_phrase:(threat.indicator.type:file))),(%27$state%27:(store:appState),meta:(alias:!n,disabled:!f,index:%27%27,key:threat.indicator.type,negate:!f,params:(query:url),type:phrase),query:(match_phrase:(threat.indicator.type:url)))),timeRange:(from:now/d,to:now/d))'; -describe('Single indicator', { tags: ['@ess'] }, () => { +// Failing: See https://github.com/elastic/kibana/issues/195804 +describe.skip('Single indicator', { tags: ['@ess'] }, () => { before(() => cy.task('esArchiverLoad', { archiveName: 'ti_indicators_data_single' })); after(() => cy.task('esArchiverUnload', { archiveName: 'ti_indicators_data_single' })); diff --git a/x-pack/test/security_solution_cypress/serverless_config.ts b/x-pack/test/security_solution_cypress/serverless_config.ts index 71a63b697187f..f3f04dda79dbb 100644 --- a/x-pack/test/security_solution_cypress/serverless_config.ts +++ b/x-pack/test/security_solution_cypress/serverless_config.ts @@ -34,7 +34,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { { product_line: 'endpoint', product_tier: 'complete' }, { product_line: 'cloud', product_tier: 'complete' }, ])}`, - `--xpack.securitySolution.enableExperimental=${JSON.stringify(['loggingRequestsEnabled'])}`, '--csp.strict=false', '--csp.warnLegacyBrowsers=false', ],