From 4a6bae46c867ced80620515717f503d5c546620c Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Fri, 4 Oct 2024 09:41:55 +0200 Subject: [PATCH] [Dataset Quality] Implement _ignored root cause identification flow (#192370) ## Summary Closes - https://github.com/elastic/kibana/issues/192471 Closes - https://github.com/elastic/kibana/issues/191055 The PR adds Flyout to the Degraded Fields inside the Dataset Quality Details page where the Root Cause of the Degraded Field is diagnosed. ## Pending Items - [x] API Tests for 1 new and 2 old API modifications - [x] E2E Tests for the Flyout ## How to test this NOTE (Below guide is for Stateful, you can do the same for serverless) - Checkout the PR using - `gh pr checkout 192370` 1. Start the FTR server using the command below ``` yarn test:ftr:server --config ./x-pack/test/functional/apps/dataset_quality/config.ts ``` 2. Go to the following path - `x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts` 3. Comment out the 2 `after` blocks present at Line - 54-56 and 414-416 4. Run the FTR runner using the command below ``` yarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts ``` Let the test run and go green 5. Navigate to `http://localhost:5620/app/management/data/data_quality/` username - `test_user` and password - `changeme` 6. Select the `degraded.dataset.rca` dataset You will have an environment ready to test the flyout different scenarios ## Demo ## Field Limit and Ignore above isse ![Field Limit Issue](https://github.com/user-attachments/assets/5908f1a8-ed85-455b-8f61-894b2fc6bb1c) ## Warning about not current quality issue ![Current Quality Issue](https://github.com/user-attachments/assets/1dd6278f-75f8-4715-bd83-8ac9784afbf7) ## Blocker There is an Elasticsearch issue on Serverless, which becomes a blocker for merging this PR https://github.com/elastic/elasticsearch-serverless/issues/2815 (cherry picked from commit 0d19367fdfad5526b5220dfdf18b4991fe6b3abd) --- .../locators/dataset_quality_details.ts | 1 + packages/kbn-optimizer/limits.yml | 2 +- .../elasticsearch/composable/template.json | 1 - .../elasticsearch/composable/template.json | 1 - .../elasticsearch/composable/template.json | 1 - .../elasticsearch/composable/template.json | 1 - .../elasticsearch/composable/template.json | 1 - .../elasticsearch/composable/template.json | 1 - .../elasticsearch/composable/template.json | 1 - x-pack/packages/kbn-data-forge/src/run.ts | 2 - .../dataset_quality_details_url_schema_v1.ts | 1 + .../dataset_quality_details/url_schema_v1.ts | 2 + .../dataset_quality/README.md | 4 +- .../dataset_quality/common/api_types.ts | 34 +- .../common/data_stream_details/types.ts | 6 + .../dataset_quality/common/es_fields/index.ts | 1 + .../dataset_quality/common/translations.ts | 98 ++++ .../dataset_quality_details.tsx | 4 +- .../degraded_field_flyout/field_info.tsx | 134 ++++-- .../degraded_field_flyout/index.tsx | 89 +++- .../degraded_fields/degraded_fields.tsx | 22 +- .../document_trends/degraded_docs/index.tsx | 27 +- .../dataset_quality_details/public_state.ts | 3 + .../dataset_quality_details/types.ts | 5 +- .../use_dataset_quality_details_state.ts | 23 +- .../public/hooks/use_degraded_fields.ts | 88 +++- .../data_stream_details_client.ts | 31 +- .../services/data_stream_details/types.ts | 8 +- .../public/services/telemetry/types.ts | 1 + .../defaults.ts | 1 + .../state_machine.ts | 306 ++++++++----- .../types.ts | 59 ++- .../get_data_stream_details/index.ts | 3 + .../get_datastream_mappings.ts | 117 +++++ .../get_datastream_settings.ts | 43 ++ .../get_degraded_field_analysis/index.ts | 52 +++ .../data_streams/get_degraded_fields/index.ts | 12 +- .../server/routes/data_streams/routes.ts | 34 ++ .../utils/create_dataset_quality_es_client.ts | 14 +- .../server/utils/to_boolean.ts | 13 + .../dataset_quality/degraded_field_analyze.ts | 157 +++++++ .../observability/dataset_quality/es_utils.ts | 41 ++ .../observability/dataset_quality/index.ts | 3 +- .../{integrations => }/integrations.ts | 4 +- .../deployment_agnostic/services/index.ts | 2 + .../services/logs_synthtrace_es_client.ts | 19 + .../data_streams/data_stream_settings.spec.ts | 37 +- .../data_streams/degraded_fields.spec.ts | 53 +++ .../tests/data_streams/es_utils.ts | 33 ++ ...t_quality_details_degraded_field_flyout.ts | 108 ----- .../dataset_quality/degraded_field_flyout.ts | 419 ++++++++++++++++++ .../functional/apps/dataset_quality/index.ts | 2 +- .../page_objects/dataset_quality.ts | 27 ++ .../data_stream_settings.ts | 15 +- .../utils/data_stream.ts | 33 ++ ...t_quality_details_degraded_field_flyout.ts | 106 ----- .../dataset_quality/degraded_field_flyout.ts | 417 +++++++++++++++++ .../observability/dataset_quality/index.ts | 2 +- 58 files changed, 2280 insertions(+), 445 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_mappings.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/utils/to_boolean.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts rename x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/{integrations => }/integrations.ts (96%) create mode 100644 x-pack/test/api_integration/deployment_agnostic/services/logs_synthtrace_es_client.ts delete mode 100644 x-pack/test/functional/apps/dataset_quality/dataset_quality_details_degraded_field_flyout.ts create mode 100644 x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details_degraded_field_flyout.ts create mode 100644 x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts diff --git a/packages/deeplinks/observability/locators/dataset_quality_details.ts b/packages/deeplinks/observability/locators/dataset_quality_details.ts index c53cabcf6dbb2..032d5d65f5665 100644 --- a/packages/deeplinks/observability/locators/dataset_quality_details.ts +++ b/packages/deeplinks/observability/locators/dataset_quality_details.ts @@ -42,4 +42,5 @@ export interface DataQualityDetailsLocatorParams extends SerializableRecord { table?: DegradedFieldsTable; }; expandedDegradedField?: string; + showCurrentQualityIssues?: boolean; } diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 8af1cb4460ecb..bd0234eaa87ec 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -27,7 +27,7 @@ pageLoadAssetSize: dashboardEnhanced: 65646 data: 454087 dataQuality: 19384 - datasetQuality: 52000 + datasetQuality: 55000 dataUsage: 30000 dataViewEditor: 28082 dataViewFieldEditor: 42021 diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_hosts/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_hosts/ecs/generated/elasticsearch/composable/template.json index 8d2a50c7437f4..2088628dc39d2 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_hosts/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_hosts/ecs/generated/elasticsearch/composable/template.json @@ -41,7 +41,6 @@ "settings": { "index": { "codec": "best_compression", - "final_pipeline": "logs@custom", "mapping": { "total_fields": { "limit": 2000 diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_logs/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_logs/ecs/generated/elasticsearch/composable/template.json index a73f04dfa2dc6..80f2ba5ff214e 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_logs/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_logs/ecs/generated/elasticsearch/composable/template.json @@ -40,7 +40,6 @@ }, "settings": { "index": { - "final_pipeline": "logs@custom", "codec": "best_compression", "mapping": { "total_fields": { diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/admin_console/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/admin_console/ecs/generated/elasticsearch/composable/template.json index 32e9c9fc701e6..e087950fd5339 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/admin_console/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/admin_console/ecs/generated/elasticsearch/composable/template.json @@ -43,7 +43,6 @@ }, "settings": { "index": { - "final_pipeline": "logs@custom", "codec": "best_compression", "mapping": { "total_fields": { diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/heartbeat/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/heartbeat/ecs/generated/elasticsearch/composable/template.json index 10ac79fcdaa69..2c43c16cf6930 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/heartbeat/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/heartbeat/ecs/generated/elasticsearch/composable/template.json @@ -38,7 +38,6 @@ }, "settings": { "index": { - "final_pipeline": "logs@custom", "codec": "best_compression", "mapping": { "total_fields": { diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/message_processor/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/message_processor/ecs/generated/elasticsearch/composable/template.json index dd33e12c38341..3635f9eb219ad 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/message_processor/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/message_processor/ecs/generated/elasticsearch/composable/template.json @@ -39,7 +39,6 @@ }, "settings": { "index": { - "final_pipeline": "logs@custom", "codec": "best_compression", "mapping": { "total_fields": { diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/mongodb/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/mongodb/ecs/generated/elasticsearch/composable/template.json index 3a29835238d94..4a0b634cbd6e2 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/mongodb/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/mongodb/ecs/generated/elasticsearch/composable/template.json @@ -39,7 +39,6 @@ }, "settings": { "index": { - "final_pipeline": "logs@custom", "codec": "best_compression", "mapping": { "total_fields": { diff --git a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json index 8afb2a15fb6ee..5f28e868e6493 100644 --- a/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json +++ b/x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json @@ -40,7 +40,6 @@ }, "settings": { "index": { - "final_pipeline": "logs@custom", "codec": "best_compression", "mapping": { "total_fields": { diff --git a/x-pack/packages/kbn-data-forge/src/run.ts b/x-pack/packages/kbn-data-forge/src/run.ts index ca9e0f355fab7..964a5da7a834f 100644 --- a/x-pack/packages/kbn-data-forge/src/run.ts +++ b/x-pack/packages/kbn-data-forge/src/run.ts @@ -13,12 +13,10 @@ import { installAssets } from './lib/install_assets'; import { indexSchedule } from './lib/index_schedule'; import { installIndexTemplate } from './lib/install_index_template'; import { indices } from './lib/indices'; -import { installDefaultIngestPipeline } from './lib/install_default_ingest_pipeline'; import { installDefaultComponentTemplate } from './lib/install_default_component_template'; export async function run(config: Config, client: Client, logger: ToolingLog) { await installDefaultComponentTemplate(config, client, logger); - await installDefaultIngestPipeline(config, client, logger); await installIndexTemplate(config, client, logger); if (config.elasticsearch.installKibanaUser) { await setupKibanaSystemUser(config, client, logger); diff --git a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts index e177a325b353b..97c7771bbf994 100644 --- a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts +++ b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts @@ -19,6 +19,7 @@ export const urlSchemaRT = rt.exact( breakdownField: rt.string, degradedFields: degradedFieldRT, expandedDegradedField: rt.string, + showCurrentQualityIssues: rt.boolean, }), ]) ); diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts index 1b08d9832c093..7b91895598eca 100644 --- a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts +++ b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts @@ -19,6 +19,7 @@ export const getStateFromUrlValue = ( degradedFields: urlValue.degradedFields, breakdownField: urlValue.breakdownField, expandedDegradedField: urlValue.expandedDegradedField, + showCurrentQualityIssues: urlValue.showCurrentQualityIssues, }); export const getUrlValueFromState = ( @@ -30,6 +31,7 @@ export const getUrlValueFromState = ( degradedFields: state.degradedFields, breakdownField: state.breakdownField, expandedDegradedField: state.expandedDegradedField, + showCurrentQualityIssues: state.showCurrentQualityIssues, v: 1, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/README.md b/x-pack/plugins/observability_solution/dataset_quality/README.md index 356393aa5237f..45883f6964cc8 100755 --- a/x-pack/plugins/observability_solution/dataset_quality/README.md +++ b/x-pack/plugins/observability_solution/dataset_quality/README.md @@ -29,7 +29,7 @@ The deployment-agnostic API tests are located in [`x-pack/test/api_integration/d node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts # run tests -node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep=$ +node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --include ./x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/$ ``` #### Start server and run test (serverless) @@ -39,7 +39,7 @@ node scripts/functional_test_runner --config x-pack/test/api_integration/deploym node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts # run tests -node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep=$ +node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --include ./x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/$ ``` ### API integration tests diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index 8bcce166b936f..bfbb2bc1cd5d1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -103,6 +103,7 @@ export const degradedFieldRt = rt.type({ y: rt.number, }) ), + indexFieldWasLastPresentIn: rt.string, }); export type DegradedField = rt.TypeOf; @@ -120,11 +121,34 @@ export const degradedFieldValuesRt = rt.type({ export type DegradedFieldValues = rt.TypeOf; -export const dataStreamSettingsRt = rt.partial({ - createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless - integration: rt.string, - datasetUserPrivileges: datasetUserPrivilegesRt, -}); +export const degradedFieldAnalysisRt = rt.intersection([ + rt.type({ + isFieldLimitIssue: rt.boolean, + fieldCount: rt.number, + totalFieldLimit: rt.number, + }), + rt.partial({ + ignoreMalformed: rt.boolean, + nestedFieldLimit: rt.number, + fieldMapping: rt.partial({ + type: rt.string, + ignore_above: rt.number, + }), + }), +]); + +export type DegradedFieldAnalysis = rt.TypeOf; + +export const dataStreamSettingsRt = rt.intersection([ + rt.type({ + lastBackingIndexName: rt.string, + }), + rt.partial({ + createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless + integration: rt.string, + datasetUserPrivileges: datasetUserPrivilegesRt, + }), +]); export type DataStreamSettings = rt.TypeOf; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index 82d6d2651be56..66b7567a2b60c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -8,3 +8,9 @@ export interface GetDataStreamIntegrationParams { integrationName: string; } + +export interface AnalyzeDegradedFieldsParams { + dataStream: string; + lastBackingIndex: string; + degradedField: string; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts index 1e67fb1c68f81..6da01c815c79e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts @@ -7,6 +7,7 @@ export const _IGNORED = '_ignored'; export const TIMESTAMP = '@timestamp'; +export const INDEX = '_index'; export const DATA_STREAM_DATASET = 'data_stream.dataset'; export const DATA_STREAM_NAMESPACE = 'data_stream.namespace'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index cdaacd2f43cff..e5b660b31de10 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -50,6 +50,13 @@ export const openInLogsExplorerText = i18n.translate( } ); +export const logsExplorerAriaText = i18n.translate( + 'xpack.datasetQuality.details.logsExplorerAriaText', + { + defaultMessage: 'Logs Explorer', + } +); + export const openInDiscoverText = i18n.translate( 'xpack.datasetQuality.details.openInDiscoverText', { @@ -57,6 +64,10 @@ export const openInDiscoverText = i18n.translate( } ); +export const discoverAriaText = i18n.translate('xpack.datasetQuality.details.discoverAriaText', { + defaultMessage: 'Discover', +}); + export const flyoutDatasetDetailsText = i18n.translate( 'xpack.datasetQuality.flyoutDatasetDetailsText', { @@ -329,6 +340,21 @@ export const overviewDegradedFieldsSectionTitle = i18n.translate( } ); +export const overviewDegradedFieldToggleSwitch = i18n.translate( + 'xpack.datasetQuality.details.degradedFieldToggleSwitch', + { + defaultMessage: 'Current quality issues only', + } +); + +export const overviewDegradedFieldToggleSwitchTooltip = i18n.translate( + 'xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip', + { + defaultMessage: + 'Enable to only show issues detected in the most recent version of the data set. Disable to show all issues detected within the configured time range.', + } +); + export const overviewDegradedFieldsSectionTitleTooltip = i18n.translate( 'xpack.datasetQuality.details.degradedFieldsSectionTooltip', { @@ -402,3 +428,75 @@ export const fieldIgnoredText = i18n.translate( defaultMessage: 'field ignored', } ); + +export const degradedFieldPotentialCauseColumnName = i18n.translate( + 'xpack.datasetQuality.details.degradedField.potentialCause', + { + defaultMessage: 'Potential cause', + } +); + +export const degradedFieldCurrentFieldLimitColumnName = i18n.translate( + 'xpack.datasetQuality.details.degradedField.currentFieldLimit', + { + defaultMessage: 'Field limit', + } +); + +export const degradedFieldMaximumCharacterLimitColumnName = i18n.translate( + 'xpack.datasetQuality.details.degradedField.maximumCharacterLimit', + { + defaultMessage: 'Maximum character length', + } +); + +export const degradedFieldCauseFieldLimitExceeded = i18n.translate( + 'xpack.datasetQuality.details.degradedField.cause.fieldLimitExceeded', + { + defaultMessage: 'field limit exceeded', + } +); + +export const degradedFieldCauseFieldLimitExceededTooltip = i18n.translate( + 'xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip', + { + defaultMessage: 'The number of fields in this index has exceeded the maximum allowed limit.', + } +); + +export const degradedFieldCauseFieldIgnored = i18n.translate( + 'xpack.datasetQuality.details.degradedField.cause.fieldIgnored', + { + defaultMessage: 'field character limit exceeded', + } +); + +export const degradedFieldCauseFieldIgnoredTooltip = i18n.translate( + 'xpack.datasetQuality.details.degradedField.cause.fieldIgnoredTooltip', + { + defaultMessage: + 'One or more values for this field exceeded the maximum allowed character length. Characters above will be ignored.', + } +); + +export const degradedFieldCauseFieldMalformed = i18n.translate( + 'xpack.datasetQuality.details.degradedField.cause.fieldMalformed', + { + defaultMessage: 'field malformed', + } +); + +export const degradedFieldCauseFieldMalformedTooltip = i18n.translate( + 'xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip', + { + defaultMessage: 'Data type for the field not set correctly.', + } +); + +export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate( + 'xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex', + { + defaultMessage: + 'This issue was detected in an older version of the dataset, but not in the most recent version.', + } +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx index d818ffe9aaf1b..59a1ae3d39d62 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx @@ -18,7 +18,7 @@ const DegradedFieldFlyout = dynamic(() => import('./degraded_field_flyout')); // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DatasetQualityDetails() { - const { isIndexNotFoundError, dataStream, expandedDegradedField } = + const { isIndexNotFoundError, dataStream, isDegradedFieldFlyoutOpen } = useDatasetQualityDetailsState(); const { startTracking } = useDatasetDetailsTelemetry(); @@ -38,7 +38,7 @@ export default function DatasetQualityDetails() {
- {expandedDegradedField && } + {isDegradedFieldFlyoutOpen && } ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx index 5e6756be96630..1e6bda781d733 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/field_info.tsx @@ -5,16 +5,16 @@ * 2.0. */ -import React, { useMemo } from 'react'; +import React from 'react'; import { EuiBadge, EuiBadgeGroup, EuiFlexGroup, EuiFlexItem, EuiHorizontalRule, - EuiSkeletonRectangle, EuiTextColor, EuiTitle, + EuiToolTip, formatNumber, } from '@elastic/eui'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; @@ -22,41 +22,42 @@ import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import { NUMBER_FORMAT } from '../../../../common/constants'; import { countColumnName, + degradedFieldCurrentFieldLimitColumnName, + degradedFieldMaximumCharacterLimitColumnName, + degradedFieldPotentialCauseColumnName, degradedFieldValuesColumnName, lastOccurrenceColumnName, } from '../../../../common/translations'; import { useDegradedFields } from '../../../hooks'; import { SparkPlot } from '../../common/spark_plot'; +import { DegradedField } from '../../../../common/api_types'; -export const DegradedFieldInfo = () => { +export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) => { const { - renderedItems, fieldFormats, - expandedDegradedField, degradedFieldValues, isDegradedFieldsLoading, - isDegradedFieldsValueLoading, + isAnalysisInProgress, + degradedFieldAnalysisResult, + degradedFieldAnalysis, } = useDegradedFields(); const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ ES_FIELD_TYPES.DATE, ]); - const fieldList = useMemo(() => { - return renderedItems.find((item) => { - return item.name === expandedDegradedField; - }); - }, [renderedItems, expandedDegradedField]); - return ( - + {countColumnName} - + { - + {lastOccurrenceColumnName} - + {dateFormatter.convert(fieldList?.lastOccurrence)} - - + + + - {degradedFieldValuesColumnName} + {degradedFieldPotentialCauseColumnName} - - - {degradedFieldValues?.values.map((value) => ( - - - {value} - - - ))} - - +
+ + + {degradedFieldAnalysisResult?.potentialCause} + + +
+ + {!isAnalysisInProgress && degradedFieldAnalysis?.isFieldLimitIssue && ( + <> + + + + {degradedFieldCurrentFieldLimitColumnName} + + + + {degradedFieldAnalysis.totalFieldLimit} + + + + + )} + + {!isAnalysisInProgress && degradedFieldAnalysisResult?.shouldDisplayValues && ( + <> + + + + {degradedFieldMaximumCharacterLimitColumnName} + + + + {degradedFieldAnalysis?.fieldMapping?.ignore_above} + + + + + + + {degradedFieldValuesColumnName} + + + + + {degradedFieldValues?.values.map((value, idx) => ( + + + {value} + + + ))} + + + + + + )} ); }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx index 84d997296b8a9..189b3ceefe37c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useMemo } from 'react'; import { EuiBadge, EuiFlyout, @@ -15,22 +15,60 @@ import { EuiText, EuiTitle, useGeneratedHtmlId, + EuiTextColor, + EuiFlexGroup, + EuiButtonIcon, + EuiToolTip, } from '@elastic/eui'; -import { useDegradedFields } from '../../../hooks'; +import { NavigationSource } from '../../../services/telemetry'; import { + useDatasetDetailsRedirectLinkTelemetry, + useDatasetQualityDetailsState, + useDegradedFields, + useRedirectLink, +} from '../../../hooks'; +import { + degradedFieldMessageIssueDoesNotExistInLatestIndex, + discoverAriaText, fieldIgnoredText, + logsExplorerAriaText, + openInDiscoverText, + openInLogsExplorerText, overviewDegradedFieldsSectionTitle, } from '../../../../common/translations'; import { DegradedFieldInfo } from './field_info'; +import { _IGNORED } from '../../../../common/es_fields'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DegradedFieldFlyout() { - const { closeDegradedFieldFlyout, expandedDegradedField } = useDegradedFields(); + const { closeDegradedFieldFlyout, expandedDegradedField, renderedItems } = useDegradedFields(); + const { dataStreamSettings, datasetDetails, timeRange } = useDatasetQualityDetailsState(); const pushedFlyoutTitleId = useGeneratedHtmlId({ prefix: 'pushedFlyoutTitle', }); + const fieldList = useMemo(() => { + return renderedItems.find((item) => { + return item.name === expandedDegradedField; + }); + }, [renderedItems, expandedDegradedField]); + + const isUserViewingTheIssueOnLatestBackingIndex = + dataStreamSettings?.lastBackingIndexName === fieldList?.indexFieldWasLastPresentIn; + + const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ + query: { language: 'kuery', query: `${_IGNORED}: ${expandedDegradedField}` }, + navigationSource: NavigationSource.DegradedFieldFlyoutHeader, + }); + + const redirectLinkProps = useRedirectLink({ + dataStreamStat: datasetDetails, + timeRangeConfig: timeRange, + query: { language: 'kuery', query: `${_IGNORED}: ${expandedDegradedField}` }, + sendTelemetry, + }); + return ( {overviewDegradedFieldsSectionTitle} - - - {expandedDegradedField} {fieldIgnoredText} - - + + + + {expandedDegradedField} {fieldIgnoredText} + + + + + + + {!isUserViewingTheIssueOnLatestBackingIndex && ( + <> + + + {degradedFieldMessageIssueDoesNotExistInLatestIndex} + + + )} - + ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx index 01956405074e3..b33bd11dbe3a6 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx @@ -15,10 +15,13 @@ import { useGeneratedHtmlId, EuiBadge, EuiBetaBadge, + EuiSwitch, } from '@elastic/eui'; import { overviewDegradedFieldsSectionTitle, overviewDegradedFieldsSectionTitleTooltip, + overviewDegradedFieldToggleSwitch, + overviewDegradedFieldToggleSwitchTooltip, overviewQualityIssuesAccordionTechPreviewBadge, } from '../../../../../common/translations'; import { DegradedFieldTable } from './table'; @@ -28,8 +31,24 @@ export function DegradedFields() { const accordionId = useGeneratedHtmlId({ prefix: overviewDegradedFieldsSectionTitle, }); + const toggleTextSwitchId = useGeneratedHtmlId({ prefix: 'toggleTextSwitch' }); - const { totalItemCount } = useDegradedFields(); + const { totalItemCount, toggleCurrentQualityIssues, showCurrentQualityIssues } = + useDegradedFields(); + + const latestBackingIndexToggle = ( + <> + + + + ); const accordionTitle = ( @@ -58,6 +77,7 @@ export function DegradedFields() { buttonContent={accordionTitle} paddingSize="none" initialIsOpen={true} + extraAction={latestBackingIndexToggle} data-test-subj="datasetQualityDetailsOverviewDocumentTrends" > diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx index a0875f1367705..05de567a6dab7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx @@ -6,7 +6,6 @@ */ import React, { useCallback, useEffect, useState } from 'react'; -import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, @@ -27,6 +26,9 @@ import type { DataViewField } from '@kbn/data-views-plugin/common'; import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; import { + discoverAriaText, + logsExplorerAriaText, + openInDiscoverText, openInLogsExplorerText, overviewDegradedDocsText, } from '../../../../../../common/translations'; @@ -130,14 +132,25 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe onBreakdownFieldChange={breakdown.onChange} /> - + + Pick< + WithDefaultControllerState, + 'timeRange' | 'breakdownField' | 'expandedDegradedField' | 'showCurrentQualityIssues' + > > & { dataStream: string; } & { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts index c24e78911b941..edd16652374a1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts @@ -51,7 +51,7 @@ export const useDatasetQualityDetailsState = () => { ); const dataStreamSettings = useSelector(service, (state) => - state.matches('initializing.dataStreamSettings.initializeIntegrations') + state.matches('initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields') ? state.context.dataStreamSettings : undefined ); @@ -59,14 +59,14 @@ export const useDatasetQualityDetailsState = () => { const integrationDetails = { integration: useSelector(service, (state) => state.matches( - 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.done' + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' ) ? state.context.integration : undefined ), dashboard: useSelector(service, (state) => state.matches( - 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.done' + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done' ) ? state.context.integrationDashboards : undefined @@ -77,7 +77,7 @@ export const useDatasetQualityDetailsState = () => { service, (state) => !state.matches( - 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.unauthorized' + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized' ) ); @@ -103,18 +103,24 @@ export const useDatasetQualityDetailsState = () => { const loadingState = useSelector(service, (state) => ({ nonAggregatableDatasetLoading: state.matches('initializing.nonAggregatableDataset.fetching'), dataStreamDetailsLoading: state.matches('initializing.dataStreamDetails.fetching'), - dataStreamSettingsLoading: state.matches('initializing.dataStreamSettings.fetching'), + dataStreamSettingsLoading: state.matches( + 'initializing.dataStreamSettings.fetchingDataStreamSettings' + ), integrationDetailsLoadings: state.matches( - 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.fetching' + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' ), integrationDetailsLoaded: state.matches( - 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.done' + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' ), integrationDashboardsLoading: state.matches( - 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.fetching' + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' ), })); + const isDegradedFieldFlyoutOpen = useSelector(service, (state) => + state.matches('initializing.degradedFieldFlyout.open') + ); + const updateTimeRange = useCallback( ({ start, end, refreshInterval }: OnRefreshProps) => { service.send({ @@ -150,5 +156,6 @@ export const useDatasetQualityDetailsState = () => { canUserAccessDashboards, canUserViewIntegrations, expandedDegradedField, + isDegradedFieldFlyoutOpen, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts index 6d52048715621..78ad0e53dd5e2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts @@ -15,6 +15,14 @@ import { } from '../../common/constants'; import { useKibanaContextForPlugin } from '../utils'; import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state'; +import { + degradedFieldCauseFieldIgnored, + degradedFieldCauseFieldIgnoredTooltip, + degradedFieldCauseFieldLimitExceeded, + degradedFieldCauseFieldLimitExceededTooltip, + degradedFieldCauseFieldMalformed, + degradedFieldCauseFieldMalformedTooltip, +} from '../../common/translations'; export type DegradedFieldSortField = keyof DegradedField; @@ -24,7 +32,10 @@ export function useDegradedFields() { services: { fieldFormats }, } = useKibanaContextForPlugin(); - const { degradedFields, expandedDegradedField } = useSelector(service, (state) => state.context); + const { degradedFields, expandedDegradedField, showCurrentQualityIssues } = useSelector( + service, + (state) => state.context + ); const { data, table } = degradedFields ?? {}; const { page, rowsPerPage, sort } = table; @@ -62,8 +73,14 @@ export function useDegradedFields() { return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage); }, [data, sort.field, sort.direction, page, rowsPerPage]); + const expandedRenderedItem = useMemo(() => { + return renderedItems.find((item) => item.name === expandedDegradedField); + }, [expandedDegradedField, renderedItems]); + const isDegradedFieldsLoading = useSelector(service, (state) => - state.matches('initializing.dataStreamDegradedFields.fetching') + state.matches( + 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching' + ) ); const closeDegradedFieldFlyout = useCallback( @@ -82,14 +99,71 @@ export function useDegradedFields() { [expandedDegradedField, service] ); + const toggleCurrentQualityIssues = useCallback(() => { + service.send('TOGGLE_CURRENT_QUALITY_ISSUES'); + }, [service]); + const degradedFieldValues = useSelector(service, (state) => - state.matches('initializing.initializeFixItFlow.ignoredValues.done') + state.matches('initializing.degradedFieldFlyout.open.ignoredValues.done') ? state.context.degradedFieldValues : undefined ); + const degradedFieldAnalysis = useSelector(service, (state) => + state.matches('initializing.degradedFieldFlyout.open.analyze.done') + ? state.context.degradedFieldAnalysis + : undefined + ); + + // This piece only cater field limit issue at the moment. + // In future this will cater the other 2 reasons as well + const degradedFieldAnalysisResult = useMemo(() => { + if (!degradedFieldAnalysis) { + return undefined; + } + + // 1st check if it's a field limit issue + if (degradedFieldAnalysis.isFieldLimitIssue) { + return { + potentialCause: degradedFieldCauseFieldLimitExceeded, + tooltipContent: degradedFieldCauseFieldLimitExceededTooltip, + shouldDisplayMitigation: true, + shouldDisplayValues: false, + }; + } + + // 2nd check if it's a ignored above issue + const fieldMapping = degradedFieldAnalysis.fieldMapping; + + if (fieldMapping && fieldMapping?.type === 'keyword' && fieldMapping?.ignore_above) { + const isAnyValueExceedingIgnoreAbove = degradedFieldValues?.values.some( + (value) => value.length > fieldMapping.ignore_above! + ); + if (isAnyValueExceedingIgnoreAbove) { + return { + potentialCause: degradedFieldCauseFieldIgnored, + tooltipContent: degradedFieldCauseFieldIgnoredTooltip, + shouldDisplayMitigation: false, + shouldDisplayValues: true, + }; + } + } + + // 3rd check if its a ignore_malformed issue. There is no check, at the moment. + return { + potentialCause: degradedFieldCauseFieldMalformed, + tooltipContent: degradedFieldCauseFieldMalformedTooltip, + shouldDisplayMitigation: false, + shouldDisplayValues: false, + }; + }, [degradedFieldAnalysis, degradedFieldValues]); + const isDegradedFieldsValueLoading = useSelector(service, (state) => { - return !state.matches('initializing.initializeFixItFlow.ignoredValues.done'); + return state.matches('initializing.degradedFieldFlyout.open.ignoredValues.fetching'); + }); + + const isAnalysisInProgress = useSelector(service, (state) => { + return state.matches('initializing.degradedFieldFlyout.open.analyze.fetching'); }); return { @@ -105,5 +179,11 @@ export function useDegradedFields() { closeDegradedFieldFlyout, degradedFieldValues, isDegradedFieldsValueLoading, + isAnalysisInProgress, + degradedFieldAnalysis, + degradedFieldAnalysisResult, + toggleCurrentQualityIssues, + showCurrentQualityIssues, + expandedRenderedItem, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 9572b4419cbe7..9175d06e105b4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,6 +8,8 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { + DegradedFieldAnalysis, + degradedFieldAnalysisRt, DegradedFieldValues, degradedFieldValuesRt, getDataStreamDegradedFieldsResponseRt, @@ -32,7 +34,10 @@ import { } from '../../../common/data_streams_stats'; import { IDataStreamDetailsClient } from './types'; import { Integration } from '../../../common/data_streams_stats/integration'; -import { GetDataStreamIntegrationParams } from '../../../common/data_stream_details/types'; +import { + AnalyzeDegradedFieldsParams, + GetDataStreamIntegrationParams, +} from '../../../common/data_stream_details/types'; import { DatasetQualityError } from '../../../common/errors'; export class DataStreamDetailsClient implements IDataStreamDetailsClient { @@ -167,4 +172,28 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { if (integration) return Integration.create(integration); } + + public async analyzeDegradedField({ + dataStream, + degradedField, + lastBackingIndex, + }: AnalyzeDegradedFieldsParams): Promise { + const response = await this.http + .get( + `/internal/dataset_quality/data_streams/${dataStream}/degraded_field/${degradedField}/analyze`, + { query: { lastBackingIndex } } + ) + .catch((error) => { + throw new DatasetQualityError( + `Failed to analyze degraded field: ${degradedField} for datastream: ${dataStream}`, + error + ); + }); + + return decodeOrThrow( + degradedFieldAnalysisRt, + (message: string) => + new DatasetQualityError(`Failed to decode the analysis response: ${message}`) + )(response); + } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index 51fed7525bfc9..a2f7db99e5af1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -17,8 +17,11 @@ import { DegradedFieldResponse, GetDataStreamDegradedFieldValuesPathParams, } from '../../../common/data_streams_stats'; -import { GetDataStreamIntegrationParams } from '../../../common/data_stream_details/types'; -import { Dashboard, DegradedFieldValues } from '../../../common/api_types'; +import { + AnalyzeDegradedFieldsParams, + GetDataStreamIntegrationParams, +} from '../../../common/data_stream_details/types'; +import { Dashboard, DegradedFieldAnalysis, DegradedFieldValues } from '../../../common/api_types'; export type DataStreamDetailsServiceSetup = void; @@ -43,4 +46,5 @@ export interface IDataStreamDetailsClient { getDataStreamIntegration( params: GetDataStreamIntegrationParams ): Promise; + analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/types.ts index f1a41ffc666cc..a0c86f6a5bd94 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/types.ts @@ -37,6 +37,7 @@ export enum NavigationSource { Trend = 'trend', Table = 'table', ActionMenu = 'action_menu', + DegradedFieldFlyoutHeader = 'degraded_field_flyout_header', } export interface WithTrackingId { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts index 024e49a9b83f4..26a51014b3abb 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts @@ -29,4 +29,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH, }, + showCurrentQualityIssues: false, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 86dbc879093f2..352aff140c275 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; +import { assign, createMachine, DoneInvokeEvent, InterpreterFrom, raise } from 'xstate'; import { getDateISORange } from '@kbn/timerange'; import type { IToasts } from '@kbn/core-notifications-browser'; import { @@ -21,6 +21,7 @@ import { Dashboard, DataStreamDetails, DataStreamSettings, + DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, @@ -47,13 +48,8 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( id: 'DatasetQualityDetailsController', context: initialContext, predictableActionArguments: true, - initial: 'uninitialized', + initial: 'initializing', states: { - uninitialized: { - always: { - target: 'initializing', - }, - }, initializing: { type: 'parallel', states: { @@ -145,58 +141,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( done: {}, }, }, - dataStreamDegradedFields: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'loadDegradedFields', - onDone: { - target: 'done', - actions: ['storeDegradedFields'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', - }, - { - target: 'done', - }, - ], - }, - }, - done: { - on: { - UPDATE_TIME_RANGE: { - target: 'fetching', - actions: ['resetDegradedFieldPageAndRowsPerPage'], - }, - UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { - target: 'done', - actions: ['storeDegradedFieldTableOptions'], - }, - OPEN_DEGRADED_FIELD_FLYOUT: { - target: - '#DatasetQualityDetailsController.initializing.initializeFixItFlow.ignoredValues', - actions: ['storeExpandedDegradedField'], - }, - CLOSE_DEGRADED_FIELD_FLYOUT: { - target: 'done', - actions: ['storeExpandedDegradedField'], - }, - }, - }, - }, - }, dataStreamSettings: { - initial: 'fetching', + initial: 'fetchingDataStreamSettings', states: { - fetching: { + fetchingDataStreamSettings: { invoke: { src: 'loadDataStreamSettings', onDone: { - target: 'initializeIntegrations', + target: 'loadingIntegrationsAndDegradedFields', actions: ['storeDataStreamSettings'], }, onError: [ @@ -211,9 +163,53 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, }, - initializeIntegrations: { + loadingIntegrationsAndDegradedFields: { type: 'parallel', states: { + dataStreamDegradedFields: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFields', + onDone: { + target: 'done', + actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], + }, + }, + done: { + on: { + UPDATE_TIME_RANGE: { + target: 'fetching', + actions: ['resetDegradedFieldPageAndRowsPerPage'], + }, + UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { + target: 'done', + actions: ['storeDegradedFieldTableOptions'], + }, + OPEN_DEGRADED_FIELD_FLYOUT: { + target: + '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', + actions: ['storeExpandedDegradedField'], + }, + TOGGLE_CURRENT_QUALITY_ISSUES: { + target: 'fetching', + actions: ['toggleCurrentQualityIssues'], + }, + }, + }, + }, + }, integrationDetails: { initial: 'fetching', states: { @@ -230,9 +226,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, }, }, - done: { - type: 'final', - }, + done: {}, }, }, integrationDashboards: { @@ -257,61 +251,115 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, }, - done: { - type: 'final', - }, + done: {}, unauthorized: { type: 'final', }, }, }, }, - }, - done: { - on: { - UPDATE_TIME_RANGE: { - target: 'fetching', - actions: ['resetDegradedFieldPageAndRowsPerPage'], - }, + onDone: { + target: 'done', }, }, + done: {}, + }, + on: { + UPDATE_TIME_RANGE: { + target: '.fetchingDataStreamSettings', + }, }, }, - initializeFixItFlow: { - initial: 'closed', - type: 'parallel', + degradedFieldFlyout: { + initial: 'pending', states: { - ignoredValues: { - initial: 'fetching', + pending: { + always: [ + { + target: 'closed', + cond: 'hasNoDegradedFieldsSelected', + }, + ], + }, + open: { + type: 'parallel', states: { - fetching: { - invoke: { - src: 'loadDegradedFieldValues', - onDone: { - target: 'done', - actions: ['storeDegradedFieldValues'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', - }, - { - target: 'done', + ignoredValues: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFieldValues', + onDone: { + target: 'done', + actions: ['storeDegradedFieldValues'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], }, - ], + }, + done: {}, }, }, - done: { - on: { - UPDATE_TIME_RANGE: { - target: 'fetching', + analyze: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'analyzeDegradedField', + onDone: { + target: 'done', + actions: ['storeDegradedFieldAnalysis'], + }, + onError: { + target: 'done', + }, + }, }, + done: {}, }, }, }, + on: { + CLOSE_DEGRADED_FIELD_FLYOUT: { + target: 'closed', + actions: ['storeExpandedDegradedField'], + }, + UPDATE_TIME_RANGE: { + target: + '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', + }, + }, + }, + closed: { + on: { + OPEN_DEGRADED_FIELD_FLYOUT: { + target: + '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', + actions: ['storeExpandedDegradedField'], + }, + }, }, }, + on: { + DEGRADED_FIELDS_LOADED: [ + { + target: '.open', + cond: 'shouldOpenFlyout', + }, + { + target: '.closed', + actions: ['storeExpandedDegradedField'], + }, + ], + }, }, }, }, @@ -370,6 +418,13 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( } : {}; }), + storeDegradedFieldAnalysis: assign((_, event: DoneInvokeEvent) => { + return 'data' in event + ? { + degradedFieldAnalysis: event.data, + } + : {}; + }), storeDegradedFieldTableOptions: assign((context, event) => { return 'degraded_field_criteria' in event ? { @@ -380,11 +435,17 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( } : {}; }), - storeExpandedDegradedField: assign((context, event) => { + storeExpandedDegradedField: assign((_, event) => { return { expandedDegradedField: 'fieldName' in event ? event.fieldName : undefined, }; }), + toggleCurrentQualityIssues: assign((context) => { + return { + showCurrentQualityIssues: !context.showCurrentQualityIssues, + }; + }), + raiseDegradedFieldsLoaded: raise('DEGRADED_FIELDS_LOADED'), resetDegradedFieldPageAndRowsPerPage: assign((context, _event) => ({ degradedFields: { ...context.degradedFields, @@ -442,6 +503,19 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( false ); }, + shouldOpenFlyout: (context) => { + return ( + Boolean(context.expandedDegradedField) && + Boolean( + context.degradedFields.data?.some( + (field) => field.name === context.expandedDegradedField + ) + ) + ); + }, + hasNoDegradedFieldsSelected: (context) => { + return !Boolean(context.expandedDegradedField); + }, }, } ); @@ -524,18 +598,46 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ loadDegradedFields: (context) => { const { startDate: start, endDate: end } = getDateISORange(context.timeRange); - return dataStreamDetailsClient.getDataStreamDegradedFields({ - dataStream: context.dataStream, - start, - end, - }); + if (!context?.isNonAggregatable) { + return dataStreamDetailsClient.getDataStreamDegradedFields({ + dataStream: + context.showCurrentQualityIssues && + 'dataStreamSettings' in context && + context.dataStreamSettings + ? context.dataStreamSettings.lastBackingIndexName + : context.dataStream, + start, + end, + }); + } + + return Promise.resolve(); }, loadDegradedFieldValues: (context) => { - return dataStreamDetailsClient.getDataStreamDegradedFieldValues({ - dataStream: context.dataStream, - degradedField: context.expandedDegradedField!, - }); + if ('expandedDegradedField' in context && context.expandedDegradedField) { + return dataStreamDetailsClient.getDataStreamDegradedFieldValues({ + dataStream: context.dataStream, + degradedField: context.expandedDegradedField, + }); + } + return Promise.resolve(); + }, + analyzeDegradedField: (context) => { + if (context?.degradedFields?.data?.length) { + const selectedDegradedField = context.degradedFields.data.find( + (field) => field.name === context.expandedDegradedField + ); + + if (selectedDegradedField) { + return dataStreamDetailsClient.analyzeDegradedField({ + dataStream: context.dataStream, + degradedField: context.expandedDegradedField!, + lastBackingIndex: selectedDegradedField.indexFieldWasLastPresentIn, + }); + } + } + return Promise.resolve(); }, loadDataStreamSettings: (context) => { return dataStreamDetailsClient.getDataStreamSettings({ diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index 3d57987743d68..cdf3bfa579e55 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -12,6 +12,7 @@ import { DataStreamDetails, DataStreamSettings, DegradedField, + DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, @@ -40,11 +41,13 @@ export interface WithDefaultControllerState { dataStream: string; degradedFields: DegradedFieldsTableConfig; timeRange: TimeRangeConfig; + showCurrentQualityIssues: boolean; breakdownField?: string; isBreakdownFieldEcs?: boolean; isIndexNotFoundError?: boolean; integration?: Integration; expandedDegradedField?: string; + isNonAggregatable?: boolean; } export interface WithDataStreamDetails { @@ -80,24 +83,29 @@ export interface WithDegradedFieldValues { degradedFieldValues: DegradedFieldValues; } +export interface WithDegradeFieldAnalysis { + degradedFieldAnalysis: DegradedFieldAnalysis; +} + export type DefaultDatasetQualityDetailsContext = Pick< WithDefaultControllerState, - 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' + 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues' >; export type DatasetQualityDetailsControllerTypeState = | { value: | 'initializing' - | 'uninitialized' | 'initializing.nonAggregatableDataset.fetching' - | 'initializing.dataStreamDegradedFields.fetching' - | 'initializing.dataStreamSettings.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching' + | 'initializing.dataStreamSettings.fetchingDataStreamSettings' | 'initializing.dataStreamDetails.fetching'; context: WithDefaultControllerState; } | { - value: 'initializing.nonAggregatableDataset.done'; + value: + | 'initializing.nonAggregatableDataset.done' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching'; context: WithDefaultControllerState & WithNonAggregatableDatasetStatus; } | { @@ -113,29 +121,44 @@ export type DatasetQualityDetailsControllerTypeState = context: WithDefaultControllerState & WithBreakdownInEcsCheck; } | { - value: 'initializing.dataStreamDegradedFields.done'; - context: WithDefaultControllerState & WithDegradedFieldsData; + value: 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.done'; + context: WithDefaultControllerState & + WithNonAggregatableDatasetStatus & + WithDegradedFieldsData; } | { - value: 'initializing.initializeFixItFlow.ignoredValues.fetching'; + value: + | 'initializing.degradedFieldFlyout.open.ignoredValues.fetching' + | 'initializing.degradedFieldFlyout.open.analyze.fetching'; context: WithDefaultControllerState & WithDegradedFieldsData; } | { - value: 'initializing.initializeFixItFlow.ignoredValues.done'; + value: 'initializing.degradedFieldFlyout.open.ignoredValues.done'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues; } + | { + value: 'initializing.degradedFieldFlyout.open.analyze.done'; + context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis; + } + | { + value: 'initializing.degradedFieldFlyout.open'; + context: WithDefaultControllerState & + WithDegradedFieldsData & + WithDegradedFieldValues & + WithDegradeFieldAnalysis; + } | { value: - | 'initializing.dataStreamSettings.initializeIntegrations' - | 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.fetching' - | 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.fetching' - | 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.unauthorized'; + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'; context: WithDefaultControllerState & WithDataStreamSettings; } | { value: - | 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.done' - | 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.done'; + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' + | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'; context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration; }; @@ -154,6 +177,9 @@ export type DatasetQualityDetailsControllerEvent = | { type: 'CLOSE_DEGRADED_FIELD_FLYOUT'; } + | { + type: 'DEGRADED_FIELDS_LOADED'; + } | { type: 'BREAKDOWN_FIELD_CHANGE'; breakdownField: string | undefined; @@ -170,4 +196,5 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent; + | DoneInvokeEvent + | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index c24ac84b10772..fd117d65ac99d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -36,12 +36,15 @@ export async function getDataStreamSettings({ dataStreamService.getMatchingDataStreams(esClient, dataStream), datasetQualityPrivileges.getDatasetPrivileges(esClient, dataStream), ]); + const integration = dataStreamInfo?._meta?.package?.name; + const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0]; return { createdOn, integration, datasetUserPrivileges, + lastBackingIndexName: lastBackingIndex?.index_name, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_mappings.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_mappings.ts new file mode 100644 index 0000000000000..865110c028a26 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_mappings.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MappingTypeMapping, + MappingProperty, + PropertyName, +} from '@elastic/elasticsearch/lib/api/types'; +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; + +export interface DataStreamMappingResponse { + fieldCount: number; + fieldPresent: boolean; + fieldMapping?: { + type?: string; + ignore_above?: number; + }; +} + +type MappingWithProperty = MappingTypeMapping & { + properties: Record; +}; + +type MappingWithFields = MappingTypeMapping & { + fields: Record; +}; +export async function getDataStreamMapping({ + field, + datasetQualityESClient, + dataStream, + lastBackingIndex, +}: { + field: string; + datasetQualityESClient: DatasetQualityESClient; + dataStream: string; + lastBackingIndex: string; +}): Promise { + const mappings = await datasetQualityESClient.mappings({ index: dataStream }); + const properties = mappings[lastBackingIndex]?.mappings?.properties; + const { count: fieldCount, capturedMapping: mapping } = countFields(properties ?? {}, field); + const fieldPresent = mapping !== undefined; + const fieldMapping = fieldPresent + ? { + type: mapping?.type, + ignore_above: (mapping as any)?.ignore_above, + } + : undefined; + + return { + fieldCount, + fieldPresent, + fieldMapping, + }; +} + +function isNestedProperty(property: MappingProperty): property is MappingWithProperty { + return 'properties' in property && property.properties !== undefined; +} + +function isNestedField(property: MappingProperty): property is MappingWithFields { + return 'fields' in property && property.fields !== undefined; +} + +function countFields( + mappings: Record, + captureField?: string, + prefix = '' +): { count: number; capturedMapping?: any } { + let fieldCount = 0; + let capturedMapping; + + for (const field in mappings) { + if (Object.prototype.hasOwnProperty.call(mappings, field)) { + const mappingField = mappings[field]; + const currentPath = [prefix, field].filter(Boolean).join('.'); + + // Capture the value if the current path matches the captureField + if (captureField && currentPath === captureField) { + capturedMapping = mappingField; + } + + fieldCount++; // Count the current field + + // If there are properties, recursively count nested fields + if (isNestedProperty(mappingField)) { + const { count, capturedMapping: nestedCapturedValue } = countFields( + mappingField.properties, + captureField, + currentPath + ); + fieldCount += count; + if (nestedCapturedValue !== undefined) { + capturedMapping = nestedCapturedValue; + } + } + + // If there are fields, recursively count nested fields + if (isNestedField(mappingField)) { + const { count, capturedMapping: nestedCapturedValue } = countFields( + mappingField.fields, + captureField, + currentPath + ); + fieldCount += count; + if (nestedCapturedValue !== undefined) { + capturedMapping = nestedCapturedValue; + } + } + } + } + + return { count: fieldCount, capturedMapping }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts new file mode 100644 index 0000000000000..433086c0b3e52 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/get_datastream_settings.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; +import { toBoolean } from '../../../utils/to_boolean'; + +export interface DataStreamSettingResponse { + nestedFieldLimit?: number; + totalFieldLimit: number; + ignoreDynamicBeyondLimit?: boolean; + ignoreMalformed?: boolean; +} + +const DEFAULT_FIELD_LIMIT = 1000; +const DEFAULT_NESTED_FIELD_LIMIT = 50; + +export async function getDataStreamSettings({ + datasetQualityESClient, + dataStream, + lastBackingIndex, +}: { + datasetQualityESClient: DatasetQualityESClient; + dataStream: string; + lastBackingIndex: string; +}): Promise { + const settings = await datasetQualityESClient.settings({ index: dataStream }); + const indexSettings = settings[lastBackingIndex]?.settings?.index?.mapping; + + return { + nestedFieldLimit: indexSettings?.nested_fields?.limit + ? Number(indexSettings?.nested_fields?.limit) + : DEFAULT_NESTED_FIELD_LIMIT, + totalFieldLimit: indexSettings?.total_fields?.limit + ? Number(indexSettings?.total_fields?.limit) + : DEFAULT_FIELD_LIMIT, + ignoreDynamicBeyondLimit: toBoolean(indexSettings?.total_fields?.ignore_dynamic_beyond_limit), + ignoreMalformed: toBoolean(indexSettings?.ignore_malformed), + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts new file mode 100644 index 0000000000000..a0e7606b475b2 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_field_analysis/index.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { DegradedFieldAnalysis } from '../../../../common/api_types'; +import { createDatasetQualityESClient } from '../../../utils'; +import { getDataStreamMapping } from './get_datastream_mappings'; +import { getDataStreamSettings } from './get_datastream_settings'; + +// TODO: The API should also in future return some analysis around the ignore_malformed check. +// As this check is expensive and steps are not very concrete, its not done for the initial iteration +export async function analyzeDegradedField({ + esClient, + dataStream, + degradedField, + lastBackingIndex, +}: { + esClient: ElasticsearchClient; + dataStream: string; + degradedField: string; + lastBackingIndex: string; +}): Promise { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const [ + { fieldCount, fieldPresent, fieldMapping }, + { nestedFieldLimit, totalFieldLimit, ignoreDynamicBeyondLimit, ignoreMalformed }, + ] = await Promise.all([ + getDataStreamMapping({ + datasetQualityESClient, + dataStream, + field: degradedField, + lastBackingIndex, + }), + getDataStreamSettings({ datasetQualityESClient, dataStream, lastBackingIndex }), + ]); + + return { + isFieldLimitIssue: Boolean( + !fieldPresent && ignoreDynamicBeyondLimit && fieldCount === totalFieldLimit + ), + fieldCount, + fieldMapping, + totalFieldLimit, + ignoreMalformed, + nestedFieldLimit, + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts index 2949a3aa99d68..0bb0b6a695fef 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts @@ -10,7 +10,7 @@ import { rangeQuery, existsQuery } from '@kbn/observability-plugin/server'; import { DegradedFieldResponse } from '../../../../common/api_types'; import { MAX_DEGRADED_FIELDS } from '../../../../common/constants'; import { createDatasetQualityESClient } from '../../../utils'; -import { _IGNORED, TIMESTAMP } from '../../../../common/es_fields'; +import { _IGNORED, INDEX, TIMESTAMP } from '../../../../common/es_fields'; import { getFieldIntervalInSeconds } from './get_interval'; export async function getDegradedFields({ @@ -43,6 +43,15 @@ export async function getDegradedFields({ field: TIMESTAMP, }, }, + index: { + terms: { + size: 1, + field: INDEX, + order: { + _key: 'desc', + }, + }, + }, timeSeries: { date_histogram: { field: TIMESTAMP, @@ -80,6 +89,7 @@ export async function getDegradedFields({ x: timeSeriesBucket.key, y: timeSeriesBucket.doc_count, })), + indexFieldWasLastPresentIn: bucket.index.buckets[0].key as string, })) ?? [], }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 869e60c6bfaa3..047004d58a6a2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -15,6 +15,7 @@ import { DegradedFieldResponse, DatasetUserPrivileges, DegradedFieldValues, + DegradedFieldAnalysis, } from '../../../common/api_types'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; @@ -26,6 +27,7 @@ import { getDegradedDocsPaginated } from './get_degraded_docs'; import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; import { getDegradedFields } from './get_degraded_fields'; import { getDegradedFieldValues } from './get_degraded_field_values'; +import { analyzeDegradedField } from './get_degraded_field_analysis'; import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; const statsRoute = createDatasetQualityServerRoute({ @@ -291,6 +293,37 @@ const dataStreamDetailsRoute = createDatasetQualityServerRoute({ }, }); +const analyzeDegradedFieldRoute = createDatasetQualityServerRoute({ + endpoint: + 'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_field/{degradedField}/analyze', + params: t.type({ + path: t.type({ + dataStream: t.string, + degradedField: t.string, + }), + query: t.type({ + lastBackingIndex: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const degradedFieldAnalysis = await analyzeDegradedField({ + esClient, + dataStream: params.path.dataStream, + degradedField: params.path.degradedField, + lastBackingIndex: params.query.lastBackingIndex, + }); + + return degradedFieldAnalysis; + }, +}); + export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, @@ -300,4 +333,5 @@ export const dataStreamsRouteRepository = { ...degradedFieldValuesRoute, ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, + ...analyzeDegradedFieldRoute, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 414c313ee373c..baa2403690fd8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -7,7 +7,13 @@ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { FieldCapsRequest, FieldCapsResponse, Indices } from '@elastic/elasticsearch/lib/api/types'; +import { + FieldCapsRequest, + FieldCapsResponse, + Indices, + IndicesGetMappingResponse, + IndicesGetSettingsResponse, +} from '@elastic/elasticsearch/lib/api/types'; type DatasetQualityESSearchParams = ESSearchRequest & { size: number; @@ -35,5 +41,11 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { async fieldCaps(params: FieldCapsRequest): Promise { return esClient.fieldCaps(params) as Promise; }, + async mappings(params: { index: string }): Promise { + return esClient.indices.getMapping(params); + }, + async settings(params: { index: string }): Promise { + return esClient.indices.getSettings(params); + }, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/to_boolean.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/to_boolean.ts new file mode 100644 index 0000000000000..395b2ee3ca645 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/to_boolean.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function toBoolean(value?: string | boolean): boolean { + if (typeof value === 'string') { + return value.toLowerCase() === 'true'; + } + return Boolean(value); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts new file mode 100644 index 0000000000000..056bde27fc33c --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/degraded_field_analyze.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import { SupertestWithRoleScopeType } from '../../../services'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './es_utils'; + +const MORE_THAN_1024_CHARS = + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const esClient = getService('es'); + const start = '2024-09-20T11:00:00.000Z'; + const end = '2024-09-20T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth.good'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream, degradedField, lastBackingIndex }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + degradedField: string; + lastBackingIndex: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials + .get( + `/internal/dataset_quality/data_streams/${dataStream}/degraded_field/${degradedField}/analyze` + ) + .query({ lastBackingIndex }); + } + + describe('Degraded field analyze', () => { + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + }); + + describe('gets limit analysis for a given datastream and degraded field', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + }) + ), + ]); + }); + + it('should return default limits and should return isFieldLimitIssue as false', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + degradedField: 'test_field', + lastBackingIndex: `${createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace, + })}-000001`, + }, + }); + + expect(resp.body.isFieldLimitIssue).to.be(false); + expect(resp.body.fieldCount).to.be(25); + expect(resp.body.fieldMapping).to.eql({ type: 'keyword', ignore_above: 1024 }); + expect(resp.body.totalFieldLimit).to.be(1000); + expect(resp.body.ignoreMalformed).to.be(true); + expect(resp.body.nestedFieldLimit).to.be(50); + }); + + it('should return updated limits and should return isFieldLimitIssue as true', async () => { + await setDataStreamSettings(esClient, dataStreamName, { + 'mapping.total_fields.limit': 25, + }); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + 'cloud.region': 'us-east-1', + }) + ), + ]); + + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + degradedField: 'cloud.region', + lastBackingIndex: `${createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace, + })}-000001`, + }, + }); + + expect(resp.body.isFieldLimitIssue).to.be(true); + expect(resp.body.fieldCount).to.be(25); + expect(resp.body.fieldMapping).to.be(undefined); // As the field limit was reached, field cannot be mapped + expect(resp.body.totalFieldLimit).to.be(25); + expect(resp.body.ignoreMalformed).to.be(true); + expect(resp.body.nestedFieldLimit).to.be(50); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts new file mode 100644 index 0000000000000..0e041781122cd --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/es_utils.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; + +function getCurrentDateFormatted() { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}.${month}.${day}`; +} + +export function createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace = 'default', +}: { + type: string; + dataset: string; + namespace: string; +}) { + return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`; +} + +export async function setDataStreamSettings( + esClient: Client, + name: string, + settings: IndicesIndexSettings +) { + return esClient.indices.putSettings({ + index: name, + settings, + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 73c58952b490b..0c660dda0a445 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -9,6 +9,7 @@ import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_cont export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) { describe('Dataset quality', () => { - loadTestFile(require.resolve('./integrations/integrations')); + loadTestFile(require.resolve('./integrations')); + loadTestFile(require.resolve('./degraded_field_analyze')); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations/integrations.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts similarity index 96% rename from x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations/integrations.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts index 0e6d319cc0ff6..910dd84bb309e 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations/integrations.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/integrations.ts @@ -8,8 +8,8 @@ import { RoleCredentials, InternalRequestHeader } from '@kbn/ftr-common-functional-services'; import expect from '@kbn/expect'; import { APIReturnType } from '@kbn/dataset-quality-plugin/common/rest'; -import { CustomIntegration } from '../../../../services/package_api'; -import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { CustomIntegration } from '../../../services/package_api'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const samlAuth = getService('samlAuth'); diff --git a/x-pack/test/api_integration/deployment_agnostic/services/index.ts b/x-pack/test/api_integration/deployment_agnostic/services/index.ts index bdaa65c139035..bea63ea216c93 100644 --- a/x-pack/test/api_integration/deployment_agnostic/services/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/services/index.ts @@ -12,6 +12,7 @@ import { deploymentAgnosticServices } from './deployment_agnostic_services'; import { PackageApiProvider } from './package_api'; import { RoleScopedSupertestProvider, SupertestWithRoleScope } from './role_scoped_supertest'; import { SloApiProvider } from './slo_api'; +import { LogsSynthtraceEsClientProvider } from './logs_synthtrace_es_client'; export type { InternalRequestHeader, @@ -28,6 +29,7 @@ export const services = { packageApi: PackageApiProvider, sloApi: SloApiProvider, roleScopedSupertest: RoleScopedSupertestProvider, + logsSynthtraceEsClient: LogsSynthtraceEsClientProvider, // create a new deployment-agnostic service and load here }; diff --git a/x-pack/test/api_integration/deployment_agnostic/services/logs_synthtrace_es_client.ts b/x-pack/test/api_integration/deployment_agnostic/services/logs_synthtrace_es_client.ts new file mode 100644 index 0000000000000..2ad02231c4490 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/services/logs_synthtrace_es_client.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createLogger, LogLevel, LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context'; + +export function LogsSynthtraceEsClientProvider({ + getService, +}: DeploymentAgnosticFtrProviderContext) { + return new LogsSynthtraceEsClient({ + client: getService('es'), + logger: createLogger(LogLevel.info), + refreshAfterIndex: true, + }); +} diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts index eafc966dc4bda..45f37b44983aa 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts @@ -15,6 +15,7 @@ import { getDataStreamSettingsOfEarliestIndex, rolloverDataStream, } from '../../utils'; +import { createBackingIndexNameWithoutVersion } from './es_utils'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); @@ -110,19 +111,30 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(resp.body).eql(defaultDataStreamPrivileges); }); - it('returns "createdOn" correctly', async () => { + it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, - `${type}-${dataset}-${namespace}` + `${type}-${integrationDataset}-${namespace}` ); const resp = await callApiAs( 'datasetQualityMonitorUser', - `${type}-${dataset}-${namespace}` + `${type}-${integrationDataset}-${namespace}` ); expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + expect(resp.body.integration).to.be('apache'); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: integrationDataset, + namespace, + })}-000001` + ); + expect(resp.body.datasetUserPrivileges).to.eql( + defaultDataStreamPrivileges.datasetUserPrivileges + ); }); - it('returns "createdOn" correctly for rolled over dataStream', async () => { + it('returns "createdOn" and "lastBackingIndexName" for rolled over dataStream', async () => { await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, @@ -133,21 +145,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { `${type}-${dataset}-${namespace}` ); expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - }); - - it('returns "createdOn" and "integration" correctly when available', async () => { - const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( - esClient, - `${type}-${integrationDataset}-${namespace}` - ); - const resp = await callApiAs( - 'datasetQualityMonitorUser', - `${type}-${integrationDataset}-${namespace}` - ); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); - expect(resp.body.integration).to.be('apache'); - expect(resp.body.datasetUserPrivileges).to.eql( - defaultDataStreamPrivileges.datasetUserPrivileges + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` ); }); diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts index fe6022067776f..100783e26e0ee 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts @@ -10,6 +10,8 @@ import expect from '@kbn/expect'; import { DegradedField } from '@kbn/dataset-quality-plugin/common/api_types'; import { DatasetQualityApiClientKey } from '../../common/config'; import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { rolloverDataStream } from '../../utils'; +import { createBackingIndexNameWithoutVersion } from './es_utils'; const MORE_THAN_1024_CHARS = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; @@ -18,6 +20,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); const synthtrace = getService('logSynthtraceEsClient'); const datasetQualityApiClient = getService('datasetQualityApiClient'); + const esClient = getService('es'); const start = '2024-05-22T08:00:00.000Z'; const end = '2024-05-23T08:02:00.000Z'; const type = 'logs'; @@ -130,6 +133,56 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(logLevelTimeSeries).to.eql(logsTimeSeriesData); }); + + it('should return the backing index where the ignored field was last seen', async () => { + await rolloverDataStream(esClient, `${type}-${degradedFieldDataset}-${namespace}`); + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a error message') + .logLevel(MORE_THAN_1024_CHARS) + .timestamp(timestamp) + .dataset(degradedFieldDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/error.log', + 'service.name': serviceName + 1, + }) + ), + ]); + + const resp = await callApiAs( + 'datasetQualityMonitorUser', + `${type}-${degradedFieldDataset}-${namespace}` + ); + + const logLevelLastBackingIndex = resp.body.degradedFields.find( + (dFields) => dFields.name === 'log.level' + )?.indexFieldWasLastPresentIn; + + const traceIdLastBackingIndex = resp.body.degradedFields.find( + (dFields) => dFields.name === 'trace.id' + )?.indexFieldWasLastPresentIn; + + expect(logLevelLastBackingIndex).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: degradedFieldDataset, + namespace, + })}-000002` + ); + expect(traceIdLastBackingIndex).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset: degradedFieldDataset, + namespace, + })}-000001` + ); + }); }); }); } diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts index 607522089952f..d4b7eb2dc0824 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts @@ -6,6 +6,7 @@ */ import { Client } from '@elastic/elasticsearch'; +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; export async function addIntegrationToLogIndexTemplate({ esClient, @@ -52,3 +53,35 @@ export async function cleanLogIndexTemplate({ esClient }: { esClient: Client }) }, }); } + +function getCurrentDateFormatted() { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}.${month}.${day}`; +} + +export function createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace = 'default', +}: { + type: string; + dataset: string; + namespace: string; +}) { + return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`; +} + +export async function setDataStreamSettings( + esClient: Client, + name: string, + settings: IndicesIndexSettings +) { + return esClient.indices.putSettings({ + index: name, + settings, + }); +} diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details_degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details_degraded_field_flyout.ts deleted file mode 100644 index 4729f5a14629d..0000000000000 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details_degraded_field_flyout.ts +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { DatasetQualityFtrProviderContext } from './config'; -import { - createDegradedFieldsRecord, - datasetNames, - defaultNamespace, - getInitialTestLogs, - ANOTHER_1024_CHARS, - MORE_THAN_1024_CHARS, -} from './data'; - -export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { - const PageObjects = getPageObjects([ - 'common', - 'navigationalSearch', - 'observabilityLogsExplorer', - 'datasetQuality', - ]); - const testSubjects = getService('testSubjects'); - const synthtrace = getService('logSynthtraceEsClient'); - const retry = getService('retry'); - const to = '2024-01-01T12:00:00.000Z'; - const degradedDatasetName = datasetNames[2]; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; - - describe('Degraded fields flyout', () => { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - }); - - after(async () => { - await synthtrace.clean(); - }); - - describe('degraded field flyout open-close', () => { - it('should open and close the flyout when user clicks on the expand button', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - }); - - await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field'); - - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); - - await PageObjects.datasetQuality.closeFlyout(); - - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); - }); - - it('should open the flyout when navigating to the page with degradedField in URL State', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - }); -} diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts new file mode 100644 index 0000000000000..517a7f2ad93fc --- /dev/null +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import moment from 'moment/moment'; +import { generateShortId, log, timerange } from '@kbn/apm-synthtrace-client'; +import { DatasetQualityFtrProviderContext } from './config'; +import { + createDegradedFieldsRecord, + datasetNames, + defaultNamespace, + getInitialTestLogs, + ANOTHER_1024_CHARS, + MORE_THAN_1024_CHARS, +} from './data'; + +export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { + const PageObjects = getPageObjects([ + 'common', + 'navigationalSearch', + 'observabilityLogsExplorer', + 'datasetQuality', + ]); + const testSubjects = getService('testSubjects'); + const synthtrace = getService('logSynthtraceEsClient'); + const retry = getService('retry'); + const to = new Date().toISOString(); + const degradedDatasetName = datasetNames[2]; + const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + + const degradedDatasetWithLimitsName = 'degraded.dataset.rca'; + const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const serviceName = 'test_service'; + const count = 5; + + describe('Degraded fields flyout', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); + + describe('degraded field flyout open-close', () => { + it('should open and close the flyout when user clicks on the expand button', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDataStreamName, + }); + + await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field'); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await PageObjects.datasetQuality.closeFlyout(); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should open the flyout when navigating to the page with degradedField in URL State', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDataStreamName, + expandedDegradedField: 'test_field', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await PageObjects.datasetQuality.closeFlyout(); + }); + }); + + describe('values exist', () => { + it('should display the degraded field values', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + MORE_THAN_1024_CHARS + ); + expect(cloudAvailabilityZoneValueExists).to.be(true); + expect(cloudAvailabilityZoneValue2Exists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + }); + + describe('testing root cause for ignored fields', () => { + before(async () => { + // Ingest Degraded Logs with 25 fields + await synthtrace.index([ + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(degradedDatasetWithLimitsName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + }) + .timestamp(timestamp) + ); + }), + ]); + + // Set Limit of 25 + await PageObjects.datasetQuality.setDataStreamSettings( + degradedDatasetWithLimitDataStreamName, + { + 'mapping.total_fields.limit': 25, + } + ); + + // Ingest Degraded Logs with 26 field + await synthtrace.index([ + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(degradedDatasetWithLimitsName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + 'cloud.region': 'us-east-1', + }) + .timestamp(timestamp) + ); + }), + ]); + + // Rollover Datastream to reset the limit to default which is 1000 + await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + + // Ingest docs with 26 fields again + await synthtrace.index([ + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(degradedDatasetWithLimitsName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + 'cloud.region': 'us-east-1', + }) + .timestamp(timestamp) + ); + }), + ]); + }); + + describe('field character limit exceeded', () => { + it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const fieldIgnoredMessageExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause', + 'field character limit exceeded' + ); + expect(fieldIgnoredMessageExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should display values when cause is "field ignored"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + MORE_THAN_1024_CHARS + ); + expect(testFieldValueExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + }); + + describe('field limit exceeded', () => { + it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await retry.tryForTime(5000, async () => { + const fieldLimitMessageExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause', + 'field limit exceeded' + ); + expect(fieldLimitMessageExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should display the limit when the cause is "field limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await retry.tryForTime(5000, async () => { + const limitExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-mappingLimit', + '25' + ); + expect(limitExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should warn users about the issue not present in latest backing index', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist + ); + }); + }); + + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(3); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(2); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/dataset_quality/index.ts b/x-pack/test/functional/apps/dataset_quality/index.ts index 4d4db08d34c88..e9c497c4407ca 100644 --- a/x-pack/test/functional/apps/dataset_quality/index.ts +++ b/x-pack/test/functional/apps/dataset_quality/index.ts @@ -15,6 +15,6 @@ export default function ({ loadTestFile }: DatasetQualityFtrProviderContext) { loadTestFile(require.resolve('./dataset_quality_table_filters')); loadTestFile(require.resolve('./dataset_quality_privileges')); loadTestFile(require.resolve('./dataset_quality_details')); - loadTestFile(require.resolve('./dataset_quality_details_degraded_field_flyout')); + loadTestFile(require.resolve('./degraded_field_flyout')); }); } diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index 437aa6e2640d5..ccd48e220064a 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -9,6 +9,7 @@ import expect from '@kbn/expect'; import querystring from 'querystring'; import rison from '@kbn/rison'; import { WebElementWrapper } from '@kbn/ftr-common-functional-ui-services'; +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; import { DATA_QUALITY_URL_STATE_KEY, datasetQualityUrlSchemaV1, @@ -77,6 +78,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv const euiSelectable = getService('selectable'); const find = getService('find'); const retry = getService('retry'); + const es = getService('es'); const selectors = { datasetQualityTable: '[data-test-subj="datasetQualityTable"]', @@ -132,6 +134,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv unifiedHistogramBreakdownSelectorSelectable: 'unifiedHistogramBreakdownSelectorSelectable', managementHome: 'managementHome', euiFlyoutCloseButton: 'euiFlyoutCloseButton', + datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist: + 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist', + datasetQualityDetailsOverviewDegradedFieldToggleSwitch: + 'datasetQualityDetailsOverviewDegradedFieldToggleSwitch', }; return { @@ -440,6 +446,27 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv return testSubjects.click(testSubjectSelectors.euiFlyoutCloseButton); }, + async setDataStreamSettings(name: string, settings: IndicesIndexSettings) { + return es.indices.putSettings({ + index: name, + settings, + }); + }, + + async rolloverDataStream(name: string) { + return es.indices.rollover({ + alias: name, + }); + }, + + async getQualityIssueSwitchState() { + const isSelected = await testSubjects.getAttribute( + testSubjectSelectors.datasetQualityDetailsOverviewDegradedFieldToggleSwitch, + 'aria-checked' + ); + return isSelected === 'true'; + }, + async parseTable(tableWrapper: WebElementWrapper, columnNamesOrIndexes: string[]) { const headerElementWrappers = await tableWrapper.findAllByCssSelector('thead th, thead td'); diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts index 52a1c51d24917..a132bc01c9720 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts @@ -14,6 +14,7 @@ import { DatasetQualityApiError, } from './common/dataset_quality_api_supertest'; import { DatasetQualityFtrContextProvider } from './common/services'; +import { createBackingIndexNameWithoutVersion } from './utils'; export default function ({ getService }: DatasetQualityFtrContextProvider) { const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient'); @@ -97,16 +98,23 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) { expect(resp.body).eql(defaultDataStreamPrivileges); }); - it('returns "createdOn" correctly', async () => { + it('returns "createdOn" and "lastBackingIndexName" correctly', async () => { const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, `${type}-${dataset}-${namespace}` ); const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace, + })}-000001` + ); }); - it('returns "createdOn" correctly for rolled over dataStream', async () => { + it('returns "createdOn" and "lastBackingIndexName" correctly for rolled over dataStream', async () => { await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( esClient, @@ -114,6 +122,9 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) { ); const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader); expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + expect(resp.body.lastBackingIndexName).to.be( + `${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002` + ); }); }); } diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts index bdf5187db0725..6fb1b043b5ce5 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts @@ -6,6 +6,7 @@ */ import { Client } from '@elastic/elasticsearch'; +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; export async function rolloverDataStream(es: Client, name: string) { return es.indices.rollover({ alias: name }); @@ -24,3 +25,35 @@ export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: str return matchingIndexesObj[matchingIndexes[0]].settings; } + +function getCurrentDateFormatted() { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}.${month}.${day}`; +} + +export function createBackingIndexNameWithoutVersion({ + type, + dataset, + namespace = 'default', +}: { + type: string; + dataset: string; + namespace: string; +}) { + return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`; +} + +export async function setDataStreamSettings( + esClient: Client, + name: string, + settings: IndicesIndexSettings +) { + return esClient.indices.putSettings({ + index: name, + settings, + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details_degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details_degraded_field_flyout.ts deleted file mode 100644 index 86632ec5a4bfc..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details_degraded_field_flyout.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { - createDegradedFieldsRecord, - datasetNames, - defaultNamespace, - getInitialTestLogs, - ANOTHER_1024_CHARS, - MORE_THAN_1024_CHARS, -} from './data'; -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export default function ({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects([ - 'common', - 'navigationalSearch', - 'observabilityLogsExplorer', - 'datasetQuality', - 'svlCommonPage', - ]); - const testSubjects = getService('testSubjects'); - const synthtrace = getService('svlLogsSynthtraceClient'); - const retry = getService('retry'); - const to = '2024-01-01T12:00:00.000Z'; - const degradedDatasetName = datasetNames[2]; - const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; - - describe('Degraded fields flyout', () => { - before(async () => { - await synthtrace.index([ - // Ingest basic logs - getInitialTestLogs({ to, count: 4 }), - // Ingest Degraded Logs - createDegradedFieldsRecord({ - to: new Date().toISOString(), - count: 2, - dataset: degradedDatasetName, - }), - ]); - await PageObjects.svlCommonPage.loginWithPrivilegedRole(); - }); - - after(async () => { - await synthtrace.clean(); - }); - - describe('degraded field flyout open-close', () => { - it('should open and close the flyout when user clicks on the expand button', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - }); - - await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field'); - - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); - - await PageObjects.datasetQuality.closeFlyout(); - }); - - it('should open the flyout when navigating to the page with degradedField in URL State', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout - ); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - - describe('values exist', () => { - it('should display the degraded field values', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - expandedDegradedField: 'test_field', - }); - - await retry.tryForTime(5000, async () => { - const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - ANOTHER_1024_CHARS - ); - const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( - 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', - MORE_THAN_1024_CHARS - ); - expect(cloudAvailabilityZoneValueExists).to.be(true); - expect(cloudAvailabilityZoneValue2Exists).to.be(true); - }); - - await PageObjects.datasetQuality.closeFlyout(); - }); - }); - }); -} diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts new file mode 100644 index 0000000000000..263dc8652ad75 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -0,0 +1,417 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import moment from 'moment'; +import { generateShortId, log, timerange } from '@kbn/apm-synthtrace-client'; +import { + createDegradedFieldsRecord, + datasetNames, + defaultNamespace, + getInitialTestLogs, + ANOTHER_1024_CHARS, + MORE_THAN_1024_CHARS, +} from './data'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects([ + 'common', + 'navigationalSearch', + 'observabilityLogsExplorer', + 'datasetQuality', + 'svlCommonPage', + ]); + const testSubjects = getService('testSubjects'); + const synthtrace = getService('svlLogsSynthtraceClient'); + const retry = getService('retry'); + const to = new Date().toISOString(); + const degradedDatasetName = datasetNames[2]; + const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + + const degradedDatasetWithLimitsName = 'degraded.dataset.rca'; + const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`; + const serviceName = 'test_service'; + const count = 5; + + describe('Degraded fields flyout', () => { + before(async () => { + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + // Ingest Degraded Logs + createDegradedFieldsRecord({ + to: new Date().toISOString(), + count: 2, + dataset: degradedDatasetName, + }), + ]); + await PageObjects.svlCommonPage.loginWithPrivilegedRole(); + }); + + after(async () => { + await synthtrace.clean(); + }); + + describe('degraded field flyout open-close', () => { + it('should open and close the flyout when user clicks on the expand button', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDataStreamName, + }); + + await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field'); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should open the flyout when navigating to the page with degradedField in URL State', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDataStreamName, + expandedDegradedField: 'test_field', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await PageObjects.datasetQuality.closeFlyout(); + }); + }); + + describe('values exist', () => { + it('should display the degraded field values', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + ANOTHER_1024_CHARS + ); + const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + MORE_THAN_1024_CHARS + ); + expect(cloudAvailabilityZoneValueExists).to.be(true); + expect(cloudAvailabilityZoneValue2Exists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + }); + + describe('testing root cause for ignored fields', () => { + before(async () => { + // Ingest Degraded Logs with 25 fields + await synthtrace.index([ + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(degradedDatasetWithLimitsName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + }) + .timestamp(timestamp) + ); + }), + ]); + + // Set Limit of 25 + await PageObjects.datasetQuality.setDataStreamSettings( + degradedDatasetWithLimitDataStreamName, + { + 'mapping.total_fields.limit': 25, + } + ); + + // Ingest Degraded Logs with 26 field + await synthtrace.index([ + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(degradedDatasetWithLimitsName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + 'cloud.region': 'us-east-1', + }) + .timestamp(timestamp) + ); + }), + ]); + + // Rollover Datastream to reset the limit to default which is 1000 + await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); + + // Ingest docs with 26 fields again + await synthtrace.index([ + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(degradedDatasetWithLimitsName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, 'hello world'], + 'cloud.region': 'us-east-1', + }) + .timestamp(timestamp) + ); + }), + ]); + }); + + describe('field character limit exceeded', () => { + it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const fieldIgnoredMessageExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause', + 'field character limit exceeded' + ); + expect(fieldIgnoredMessageExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should display values when cause is "field ignored"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + }); + + await retry.tryForTime(5000, async () => { + const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values', + MORE_THAN_1024_CHARS + ); + expect(testFieldValueExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + }); + + describe('field limit exceeded', () => { + it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await retry.tryForTime(5000, async () => { + const fieldLimitMessageExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause', + 'field limit exceeded' + ); + expect(fieldLimitMessageExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should display the limit when the cause is "field limit exceeded"', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await retry.tryForTime(5000, async () => { + const limitExists = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldValue-mappingLimit', + '25' + ); + expect(limitExists).to.be(true); + }); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should warn users about the issue not present in latest backing index', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist + ); + }); + }); + + describe('current quality issues', () => { + it('should display issues only from latest backing index when current issues toggle is on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(false); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(rows.length).to.eql(3); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + const newCurrentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(newCurrentIssuesToggleState).to.be(true); + + const newRows = + await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); + + expect(newRows.length).to.eql(2); + }); + + it('should keep the toggle on when url state says so', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + const currentIssuesToggleState = + await PageObjects.datasetQuality.getQualityIssueSwitchState(); + + expect(currentIssuesToggleState).to.be(true); + }); + + it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'test_field', + showCurrentQualityIssues: true, + }); + + // Check value in Table + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const countColumn = table['Docs count']; + expect(await countColumn.getCellTexts()).to.eql(['5', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const countValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '5' + ); + expect(countValue).to.be(true); + }); + + // Toggle the switch + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + // Check value in Table + const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable(); + const newCountColumn = newTable['Docs count']; + expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']); + + // Check value in Flyout + await retry.tryForTime(5000, async () => { + const newCountValue = await PageObjects.datasetQuality.doesTextExist( + 'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount', + '15' + ); + expect(newCountValue).to.be(true); + }); + }); + + it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + showCurrentQualityIssues: true, + }); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + + it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: degradedDatasetWithLimitDataStreamName, + expandedDegradedField: 'cloud', + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + + await testSubjects.click( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsOverviewDegradedFieldToggleSwitch + ); + + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout + ); + }); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/index.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/index.ts index 699c05de7330f..5a481ac61c757 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/index.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/index.ts @@ -15,6 +15,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./dataset_quality_table_filters')); loadTestFile(require.resolve('./dataset_quality_privileges')); loadTestFile(require.resolve('./dataset_quality_details')); - loadTestFile(require.resolve('./dataset_quality_details_degraded_field_flyout')); + loadTestFile(require.resolve('./degraded_field_flyout')); }); }