[Dataset Quality] Implement _ignored root cause identification flow (#…

…192370) ## Summary Closes - #192471 Closes - #191055 The PR adds Flyout to the Degraded Fields inside the Dataset Quality Details page where the Root Cause of the Degraded Field is diagnosed. ## Pending Items - [x] API Tests for 1 new and 2 old API modifications - [x] E2E Tests for the Flyout ## How to test this NOTE (Below guide is for Stateful, you can do the same for serverless) - Checkout the PR using - `gh pr checkout 192370` 1. Start the FTR server using the command below ``` yarn test:ftr:server --config ./x-pack/test/functional/apps/dataset_quality/config.ts ``` 2. Go to the following path - `x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts` 3. Comment out the 2 `after` blocks present at Line - 54-56 and 414-416 4. Run the FTR runner using the command below ``` yarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts ``` Let the test run and go green 5. Navigate to `http://localhost:5620/app/management/data/data_quality/` username - `test_user` and password - `changeme` 6. Select the `degraded.dataset.rca` dataset You will have an environment ready to test the flyout different scenarios ## Demo ## Field Limit and Ignore above isse ![Field Limit Issue](https://github.com/user-attachments/assets/5908f1a8-ed85-455b-8f61-894b2fc6bb1c) ## Warning about not current quality issue ![Current Quality Issue](https://github.com/user-attachments/assets/1dd6278f-75f8-4715-bd83-8ac9784afbf7) ## Blocker There is an Elasticsearch issue on Serverless, which becomes a blocker for merging this PR elastic/elasticsearch-serverless#2815 (cherry picked from commit 0d19367)
elastic · Oct 4, 2024 · 4a6bae4 · 4a6bae4
1 parent 18b2b5c
commit 4a6bae4
Show file tree

Hide file tree

Showing 58 changed files with 2,280 additions and 445 deletions.
diff --git a/packages/deeplinks/observability/locators/dataset_quality_details.ts b/packages/deeplinks/observability/locators/dataset_quality_details.ts
@@ -42,4 +42,5 @@ export interface DataQualityDetailsLocatorParams extends SerializableRecord {
     table?: DegradedFieldsTable;
   };
   expandedDegradedField?: string;
+  showCurrentQualityIssues?: boolean;
 }
diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
@@ -27,7 +27,7 @@ pageLoadAssetSize:
   dashboardEnhanced: 65646
   data: 454087
   dataQuality: 19384
-  datasetQuality: 52000
+  datasetQuality: 55000
   dataUsage: 30000
   dataViewEditor: 28082
   dataViewFieldEditor: 42021

diff --git a/...ta-forge/src/data_sources/fake_hosts/ecs/generated/elasticsearch/composable/template.json b/...ta-forge/src/data_sources/fake_hosts/ecs/generated/elasticsearch/composable/template.json
@@ -41,7 +41,6 @@
     "settings": {
       "index": {
         "codec": "best_compression",
-        "final_pipeline": "logs@custom",
         "mapping": {
           "total_fields": {
             "limit": 2000

diff --git a/...ata-forge/src/data_sources/fake_logs/ecs/generated/elasticsearch/composable/template.json b/...ata-forge/src/data_sources/fake_logs/ecs/generated/elasticsearch/composable/template.json
@@ -40,7 +40,6 @@
     },
     "settings": {
       "index": {
-        "final_pipeline": "logs@custom",
         "codec": "best_compression",
         "mapping": {
           "total_fields": {

diff --git a/...ata_sources/fake_stack/admin_console/ecs/generated/elasticsearch/composable/template.json b/...ata_sources/fake_stack/admin_console/ecs/generated/elasticsearch/composable/template.json
@@ -43,7 +43,6 @@
     },
     "settings": {
       "index": {
-        "final_pipeline": "logs@custom",
         "codec": "best_compression",
         "mapping": {
           "total_fields": {

diff --git a/...rc/data_sources/fake_stack/heartbeat/ecs/generated/elasticsearch/composable/template.json b/...rc/data_sources/fake_stack/heartbeat/ecs/generated/elasticsearch/composable/template.json
@@ -38,7 +38,6 @@
     },
     "settings": {
       "index": {
-        "final_pipeline": "logs@custom",
         "codec": "best_compression",
         "mapping": {
           "total_fields": {

diff --git a/...sources/fake_stack/message_processor/ecs/generated/elasticsearch/composable/template.json b/...sources/fake_stack/message_processor/ecs/generated/elasticsearch/composable/template.json
@@ -39,7 +39,6 @@
     },
     "settings": {
       "index": {
-        "final_pipeline": "logs@custom",
         "codec": "best_compression",
         "mapping": {
           "total_fields": {

diff --git a/.../src/data_sources/fake_stack/mongodb/ecs/generated/elasticsearch/composable/template.json b/.../src/data_sources/fake_stack/mongodb/ecs/generated/elasticsearch/composable/template.json
@@ -39,7 +39,6 @@
     },
     "settings": {
       "index": {
-        "final_pipeline": "logs@custom",
         "codec": "best_compression",
         "mapping": {
           "total_fields": {

diff --git a/.../data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json b/.../data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json
@@ -40,7 +40,6 @@
     },
     "settings": {
       "index": {
-        "final_pipeline": "logs@custom",
         "codec": "best_compression",
         "mapping": {
           "total_fields": {

diff --git a/x-pack/packages/kbn-data-forge/src/run.ts b/x-pack/packages/kbn-data-forge/src/run.ts
@@ -13,12 +13,10 @@ import { installAssets } from './lib/install_assets';
 import { indexSchedule } from './lib/index_schedule';
 import { installIndexTemplate } from './lib/install_index_template';
 import { indices } from './lib/indices';
-import { installDefaultIngestPipeline } from './lib/install_default_ingest_pipeline';
 import { installDefaultComponentTemplate } from './lib/install_default_component_template';
 
 export async function run(config: Config, client: Client, logger: ToolingLog) {
   await installDefaultComponentTemplate(config, client, logger);
-  await installDefaultIngestPipeline(config, client, logger);
   await installIndexTemplate(config, client, logger);
   if (config.elasticsearch.installKibanaUser) {
     await setupKibanaSystemUser(config, client, logger);

diff --git a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts
@@ -19,6 +19,7 @@ export const urlSchemaRT = rt.exact(
       breakdownField: rt.string,
       degradedFields: degradedFieldRT,
       expandedDegradedField: rt.string,
+      showCurrentQualityIssues: rt.boolean,
     }),
   ])
 );

diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts
@@ -19,6 +19,7 @@ export const getStateFromUrlValue = (
     degradedFields: urlValue.degradedFields,
     breakdownField: urlValue.breakdownField,
     expandedDegradedField: urlValue.expandedDegradedField,
+    showCurrentQualityIssues: urlValue.showCurrentQualityIssues,
   });
 
 export const getUrlValueFromState = (
@@ -30,6 +31,7 @@ export const getUrlValueFromState = (
     degradedFields: state.degradedFields,
     breakdownField: state.breakdownField,
     expandedDegradedField: state.expandedDegradedField,
+    showCurrentQualityIssues: state.showCurrentQualityIssues,
     v: 1,
   });
 

diff --git a/x-pack/plugins/observability_solution/dataset_quality/README.md b/x-pack/plugins/observability_solution/dataset_quality/README.md
@@ -29,7 +29,7 @@ The deployment-agnostic API tests are located in [`x-pack/test/api_integration/d
 node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
 
 # run tests
-node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep=$
+node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --include ./x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/$
 ```
 
 #### Start server and run test (serverless)
@@ -39,7 +39,7 @@ node scripts/functional_test_runner --config x-pack/test/api_integration/deploym
 node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
 
 # run tests
-node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep=$
+node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --include ./x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/$
 ```
 
 ### API integration tests

diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts
@@ -103,6 +103,7 @@ export const degradedFieldRt = rt.type({
       y: rt.number,
     })
   ),
+  indexFieldWasLastPresentIn: rt.string,
 });
 
 export type DegradedField = rt.TypeOf<typeof degradedFieldRt>;
@@ -120,11 +121,34 @@ export const degradedFieldValuesRt = rt.type({
 
 export type DegradedFieldValues = rt.TypeOf<typeof degradedFieldValuesRt>;
 
-export const dataStreamSettingsRt = rt.partial({
-  createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
-  integration: rt.string,
-  datasetUserPrivileges: datasetUserPrivilegesRt,
-});
+export const degradedFieldAnalysisRt = rt.intersection([
+  rt.type({
+    isFieldLimitIssue: rt.boolean,
+    fieldCount: rt.number,
+    totalFieldLimit: rt.number,
+  }),
+  rt.partial({
+    ignoreMalformed: rt.boolean,
+    nestedFieldLimit: rt.number,
+    fieldMapping: rt.partial({
+      type: rt.string,
+      ignore_above: rt.number,
+    }),
+  }),
+]);
+
+export type DegradedFieldAnalysis = rt.TypeOf<typeof degradedFieldAnalysisRt>;
+
+export const dataStreamSettingsRt = rt.intersection([
+  rt.type({
+    lastBackingIndexName: rt.string,
+  }),
+  rt.partial({
+    createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
+    integration: rt.string,
+    datasetUserPrivileges: datasetUserPrivilegesRt,
+  }),
+]);
 
 export type DataStreamSettings = rt.TypeOf<typeof dataStreamSettingsRt>;
 

diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts
@@ -8,3 +8,9 @@
 export interface GetDataStreamIntegrationParams {
   integrationName: string;
 }
+
+export interface AnalyzeDegradedFieldsParams {
+  dataStream: string;
+  lastBackingIndex: string;
+  degradedField: string;
+}
diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts
@@ -7,6 +7,7 @@
 
 export const _IGNORED = '_ignored';
 export const TIMESTAMP = '@timestamp';
+export const INDEX = '_index';
 
 export const DATA_STREAM_DATASET = 'data_stream.dataset';
 export const DATA_STREAM_NAMESPACE = 'data_stream.namespace';

diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts
@@ -50,13 +50,24 @@ export const openInLogsExplorerText = i18n.translate(
   }
 );
 
+export const logsExplorerAriaText = i18n.translate(
+  'xpack.datasetQuality.details.logsExplorerAriaText',
+  {
+    defaultMessage: 'Logs Explorer',
+  }
+);
+
 export const openInDiscoverText = i18n.translate(
   'xpack.datasetQuality.details.openInDiscoverText',
   {
     defaultMessage: 'Open in Discover',
   }
 );
 
+export const discoverAriaText = i18n.translate('xpack.datasetQuality.details.discoverAriaText', {
+  defaultMessage: 'Discover',
+});
+
 export const flyoutDatasetDetailsText = i18n.translate(
   'xpack.datasetQuality.flyoutDatasetDetailsText',
   {
@@ -329,6 +340,21 @@ export const overviewDegradedFieldsSectionTitle = i18n.translate(
   }
 );
 
+export const overviewDegradedFieldToggleSwitch = i18n.translate(
+  'xpack.datasetQuality.details.degradedFieldToggleSwitch',
+  {
+    defaultMessage: 'Current quality issues only',
+  }
+);
+
+export const overviewDegradedFieldToggleSwitchTooltip = i18n.translate(
+  'xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip',
+  {
+    defaultMessage:
+      'Enable to only show issues detected in the most recent version of the data set. Disable to show all issues detected within the configured time range.',
+  }
+);
+
 export const overviewDegradedFieldsSectionTitleTooltip = i18n.translate(
   'xpack.datasetQuality.details.degradedFieldsSectionTooltip',
   {
@@ -402,3 +428,75 @@ export const fieldIgnoredText = i18n.translate(
     defaultMessage: 'field ignored',
   }
 );
+
+export const degradedFieldPotentialCauseColumnName = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.potentialCause',
+  {
+    defaultMessage: 'Potential cause',
+  }
+);
+
+export const degradedFieldCurrentFieldLimitColumnName = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.currentFieldLimit',
+  {
+    defaultMessage: 'Field limit',
+  }
+);
+
+export const degradedFieldMaximumCharacterLimitColumnName = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.maximumCharacterLimit',
+  {
+    defaultMessage: 'Maximum character length',
+  }
+);
+
+export const degradedFieldCauseFieldLimitExceeded = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.cause.fieldLimitExceeded',
+  {
+    defaultMessage: 'field limit exceeded',
+  }
+);
+
+export const degradedFieldCauseFieldLimitExceededTooltip = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip',
+  {
+    defaultMessage: 'The number of fields in this index has exceeded the maximum allowed limit.',
+  }
+);
+
+export const degradedFieldCauseFieldIgnored = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.cause.fieldIgnored',
+  {
+    defaultMessage: 'field character limit exceeded',
+  }
+);
+
+export const degradedFieldCauseFieldIgnoredTooltip = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.cause.fieldIgnoredTooltip',
+  {
+    defaultMessage:
+      'One or more values for this field exceeded the maximum allowed character length. Characters above will be ignored.',
+  }
+);
+
+export const degradedFieldCauseFieldMalformed = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.cause.fieldMalformed',
+  {
+    defaultMessage: 'field malformed',
+  }
+);
+
+export const degradedFieldCauseFieldMalformedTooltip = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip',
+  {
+    defaultMessage: 'Data type for the field not set correctly.',
+  }
+);
+
+export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate(
+  'xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex',
+  {
+    defaultMessage:
+      'This issue was detected in an older version of the dataset, but not in the most recent version.',
+  }
+);
diff --git a/...ion/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx b/...ion/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx
@@ -18,7 +18,7 @@ const DegradedFieldFlyout = dynamic(() => import('./degraded_field_flyout'));
 // Allow for lazy loading
 // eslint-disable-next-line import/no-default-export
 export default function DatasetQualityDetails() {
-  const { isIndexNotFoundError, dataStream, expandedDegradedField } =
+  const { isIndexNotFoundError, dataStream, isDegradedFieldFlyoutOpen } =
     useDatasetQualityDetailsState();
   const { startTracking } = useDatasetDetailsTelemetry();
 
@@ -38,7 +38,7 @@ export default function DatasetQualityDetails() {
           <Details />
         </EuiFlexItem>
       </EuiFlexGroup>
-      {expandedDegradedField && <DegradedFieldFlyout />}
+      {isDegradedFieldFlyoutOpen && <DegradedFieldFlyout />}
     </>
   );
 }