[Dataset quality] Support failure store

x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts

@@ -56,6 +56,12 @@ export const getDataStreamDegradedDocsResponseRt = rt.type({
 
 export type DataStreamDegradedDocsResponse = rt.TypeOf<typeof getDataStreamDegradedDocsResponseRt>;
 
+export const getDataStreamFailedDocsResponseRt = rt.type({
+  failedDocs: rt.array(dataStreamDocsStatRt),
+});
+
+export type DataStreamFailedDocsResponse = rt.TypeOf<typeof getDataStreamFailedDocsResponseRt>;
+
 export const integrationDashboardRT = rt.type({
   id: rt.string,
   title: rt.string,

x-pack/plugins/observability_solution/dataset_quality/common/constants.ts

@@ -26,7 +26,7 @@ export const NONE = 'none';
 export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' };
 export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false };
 
-export const DEFAULT_DEGRADED_DOCS = {
+export const DEFAULT_QUALITY_DOC_STATS = {
   count: 0,
   percentage: 0,
 };

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts

@@ -6,12 +6,14 @@
  */
 
 import { DataStreamDocsStat } from '../api_types';
-import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants';
+import { DEFAULT_DATASET_QUALITY, DEFAULT_QUALITY_DOC_STATS } from '../constants';
 import { DataStreamType, QualityIndicators } from '../types';
 import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils';
 import { Integration } from './integration';
 import { DataStreamStatType } from './types';
 
+type QualityStat = Omit<DataStreamDocsStat, 'dataset'> & { percentage: number };
+
 export class DataStreamStat {
   rawName: string;
   type: DataStreamType;
@@ -30,6 +32,10 @@ export class DataStreamStat {
     percentage: number;
     count: number;
   };
+  failedDocs: {
+    percentage: number;
+    count: number;
+  };
 
   private constructor(dataStreamStat: DataStreamStat) {
     this.rawName = dataStreamStat.rawName;
@@ -46,6 +52,7 @@ export class DataStreamStat {
     this.quality = dataStreamStat.quality;
     this.docsInTimeRange = dataStreamStat.docsInTimeRange;
     this.degradedDocs = dataStreamStat.degradedDocs;
+    this.failedDocs = dataStreamStat.failedDocs;
   }
 
   public static create(dataStreamStat: DataStreamStatType) {
@@ -63,36 +70,45 @@ export class DataStreamStat {
       userPrivileges: dataStreamStat.userPrivileges,
       totalDocs: dataStreamStat.totalDocs,
       quality: DEFAULT_DATASET_QUALITY,
-      degradedDocs: DEFAULT_DEGRADED_DOCS,
+      degradedDocs: DEFAULT_QUALITY_DOC_STATS,
+      failedDocs: DEFAULT_QUALITY_DOC_STATS,
     };
 
     return new DataStreamStat(dataStreamStatProps);
   }
 
-  public static fromDegradedDocStat({
+  public static fromQualityStats({
+    datasetName,
     degradedDocStat,
+    failedDocStat,
     datasetIntegrationMap,
     totalDocs,
   }: {
-    degradedDocStat: DataStreamDocsStat & { percentage: number };
+    datasetName: string;
+    degradedDocStat: QualityStat;
+    failedDocStat: QualityStat;
     datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
     totalDocs: number;
   }) {
-    const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset);
+    const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName);
 
     const dataStreamStatProps = {
-      rawName: degradedDocStat.dataset,
+      rawName: datasetName,
       type,
       name: dataset,
-      title: datasetIntegrationMap[dataset]?.title || dataset,
+      title: datasetIntegrationMap[datasetName]?.title || dataset,
       namespace,
-      integration: datasetIntegrationMap[dataset]?.integration,
-      quality: mapPercentageToQuality(degradedDocStat.percentage),
+      integration: datasetIntegrationMap[datasetName]?.integration,
+      quality: mapPercentageToQuality([degradedDocStat.percentage, failedDocStat.percentage]),
       docsInTimeRange: totalDocs,
       degradedDocs: {
         percentage: degradedDocStat.percentage,
         count: degradedDocStat.count,
       },
+      failedDocs: {
+        percentage: failedDocStat.percentage,
+        count: failedDocStat.count,
+      },
     };
 
     return new DataStreamStat(dataStreamStatProps);

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts

@@ -18,6 +18,9 @@ export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse;
 export type GetDataStreamsDegradedDocsStatsParams =
   APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params'];
 export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query'];
+export type GetDataStreamsFailedDocsStatsParams =
+  APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/failed_docs`>['params'];
+export type GetDataStreamsFailedDocsStatsQuery = GetDataStreamsFailedDocsStatsParams['query'];
 
 /*
 Types for stats based in documents inside a DataStream

x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts

@@ -84,5 +84,11 @@ describe('dataset_name', () => {
         extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001')
       ).toEqual('metrics-apm.app.adservice-default');
     });
+
+    it('returns the correct index name if backing index is a failure store index', () => {
+      expect(
+        extractIndexNameFromBackingIndex('.fs-logs-elastic_agent-default-2024.11.11-000001')
+      ).toEqual('logs-elastic_agent-default');
+    });
   });
 });

x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts

@@ -40,7 +40,7 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => {
 };
 
 export const extractIndexNameFromBackingIndex = (indexString: string): string => {
-  const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
+  const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
   const match = indexString.match(pattern);
 
   return match ? match[1] : indexString;

x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts

@@ -8,10 +8,14 @@
 import { POOR_QUALITY_MINIMUM_PERCENTAGE, DEGRADED_QUALITY_MINIMUM_PERCENTAGE } from '../constants';
 import { QualityIndicators } from '../types';
 
-export const mapPercentageToQuality = (percentage: number): QualityIndicators => {
-  return percentage > POOR_QUALITY_MINIMUM_PERCENTAGE
-    ? 'poor'
-    : percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE
-    ? 'degraded'
-    : 'good';
+export const mapPercentageToQuality = (percentages: number[]): QualityIndicators => {
+  if (percentages.some((percentage) => percentage > POOR_QUALITY_MINIMUM_PERCENTAGE)) {
+    return 'poor';
+  }
+
+  if (percentages.some((percentage) => percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE)) {
+    return 'degraded';
+  }
+
+  return 'good';
 };

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx

@@ -29,20 +29,11 @@ import {
   summaryPanelQualityText,
   summaryPanelQualityTooltipText,
 } from '../../../../common/translations';
-import { mapPercentagesToQualityCounts } from '../../quality_indicator';
 
 export function DatasetsQualityIndicators() {
   const { onPageReady } = usePerformanceContext();
-  const {
-    datasetsQuality,
-    isDatasetsQualityLoading,
-    datasetsActivity,
-    numberOfDatasets,
-    numberOfDocuments,
-  } = useSummaryPanelContext();
-  const qualityCounts = mapPercentagesToQualityCounts(datasetsQuality.percentages);
-  const datasetsWithoutIgnoredField =
-    datasetsActivity.total > 0 ? datasetsActivity.total - datasetsQuality.percentages.length : 0;
+  const { datasetsQuality, isDatasetsQualityLoading, numberOfDatasets, numberOfDocuments } =
+    useSummaryPanelContext();
 
   if (!isDatasetsQualityLoading && (numberOfDatasets || numberOfDocuments)) {
     onPageReady({
@@ -66,21 +57,21 @@ export function DatasetsQualityIndicators() {
         </EuiFlexGroup>
         <EuiFlexGroup gutterSize="m" alignItems="flexEnd">
           <QualityIndicator
-            value={qualityCounts.poor}
+            value={datasetsQuality.poor}
             quality="danger"
             description={summaryPanelQualityPoorText}
             isLoading={isDatasetsQualityLoading}
           />
           <span css={verticalRule} />
           <QualityIndicator
-            value={qualityCounts.degraded}
+            value={datasetsQuality.degraded}
             quality="warning"
             description={summaryPanelQualityDegradedText}
             isLoading={isDatasetsQualityLoading}
           />
           <span css={verticalRule} />
           <QualityIndicator
-            value={qualityCounts.good + datasetsWithoutIgnoredField}
+            value={datasetsQuality.good}
             quality="success"
             description={summaryPanelQualityGoodText}
             isLoading={isDatasetsQualityLoading}

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx

@@ -37,6 +37,7 @@ import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks
 import { DegradedDocsPercentageLink } from './degraded_docs_percentage_link';
 import { TimeRangeConfig } from '../../../../common/types';
 import { DatasetQualityDetailsLink } from './dataset_quality_details_link';
+import { FailedDocsPercentageLink } from './failed_docs_percentage_link';
 
 const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', {
   defaultMessage: 'Data Set Name',
@@ -58,6 +59,10 @@ const degradedDocsColumnName = i18n.translate('xpack.datasetQuality.degradedDocs
   defaultMessage: 'Degraded Docs (%)',
 });
 
+const failedDocsColumnName = i18n.translate('xpack.datasetQuality.failedDocsColumnName', {
+  defaultMessage: 'Failed Docs (%)',
+});
+
 const datasetQualityColumnName = i18n.translate('xpack.datasetQuality.datasetQualityColumnName', {
   defaultMessage: 'Data Set Quality',
 });
@@ -112,10 +117,17 @@ const degradedDocsColumnTooltip = (
   />
 );
 
+const failedDocsColumnTooltip = (
+  <FormattedMessage
+    id="xpack.datasetQuality.failedDocsColumnTooltip"
+    defaultMessage="The percentage of documents in the failure store for your data set."
+  />
+);
+
 const datasetQualityColumnTooltip = (
   <FormattedMessage
     id="xpack.datasetQuality.datasetQualityColumnTooltip"
-    defaultMessage="Quality is based on the percentage of degraded docs in a data set. {visualQueue}"
+    defaultMessage="Quality is based on the percentage of degraded and failed docs in a data set. {visualQueue}"
     values={{
       visualQueue: (
         <EuiFlexGroup direction="column" gutterSize="xs">
@@ -273,7 +285,7 @@ export const getDatasetQualityTableColumns = ({
           </EuiToolTip>
         </EuiTableHeader>
       ),
-      field: 'degradedDocs.percentage',
+      field: 'quality',
       sortable: true,
       render: (_, dataStreamStat: DataStreamStat) => (
         <DatasetQualityIndicator isLoading={loadingDegradedStats} dataStreamStat={dataStreamStat} />
@@ -304,35 +316,61 @@ export const getDatasetQualityTableColumns = ({
     },
     {
       name: (
-        <EuiTableHeader data-test-subj="datasetQualityLastActivityColumn">
-          {lastActivityColumnName}
+        <EuiTableHeader data-test-subj="datasetQualityFailedPercentageColumn">
+          <EuiToolTip content={failedDocsColumnTooltip}>
+            <span>
+              {`${failedDocsColumnName} `}
+              <EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
+            </span>
+          </EuiToolTip>
         </EuiTableHeader>
       ),
-      field: 'lastActivity',
-      render: (timestamp: number) => (
-        <EuiSkeletonRectangle
-          width="200px"
-          height="20px"
-          borderRadius="m"
-          isLoading={loadingDataStreamStats}
-        >
-          {!isActiveDataset(timestamp) ? (
-            <EuiFlexGroup gutterSize="xs" alignItems="center">
-              <EuiText size="s">{inactiveDatasetActivityColumnDescription}</EuiText>
-              <EuiToolTip position="top" content={inactiveDatasetActivityColumnTooltip}>
-                <EuiIcon tabIndex={0} type="iInCircle" size="s" />
-              </EuiToolTip>
-            </EuiFlexGroup>
-          ) : (
-            fieldFormats
-              .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
-              .convert(timestamp)
-          )}
-        </EuiSkeletonRectangle>
-      ),
-      width: '300px',
+      field: 'failedDocs.percentage',
       sortable: true,
+      render: (_, dataStreamStat: DataStreamStat) => (
+        <FailedDocsPercentageLink
+          isLoading={loadingDegradedStats}
+          dataStreamStat={dataStreamStat}
+          timeRange={timeRange}
+        />
+      ),
+      width: '140px',
     },
+    ...(canUserMonitorDataset && canUserMonitorAnyDataStream
+      ? [
+          {
+            name: (
+              <EuiTableHeader data-test-subj="datasetQualityLastActivityColumn">
+                {lastActivityColumnName}
+              </EuiTableHeader>
+            ),
+            field: 'lastActivity',
+            render: (timestamp: number) => (
+              <EuiSkeletonRectangle
+                width="200px"
+                height="20px"
+                borderRadius="m"
+                isLoading={loadingDataStreamStats}
+              >
+                {!isActiveDataset(timestamp) ? (
+                  <EuiFlexGroup gutterSize="xs" alignItems="center">
+                    <EuiText size="s">{inactiveDatasetActivityColumnDescription}</EuiText>
+                    <EuiToolTip position="top" content={inactiveDatasetActivityColumnTooltip}>
+                      <EuiIcon tabIndex={0} type="iInCircle" size="s" />
+                    </EuiToolTip>
+                  </EuiFlexGroup>
+                ) : (
+                  fieldFormats
+                    .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
+                    .convert(timestamp)
+                )}
+              </EuiSkeletonRectangle>
+            ),
+            width: '300px',
+            sortable: true,
+          },
+        ]
+      : []),
     {
       name: actionsColumnName,
       render: (dataStreamStat: DataStreamStat) => (

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiSkeletonRectangle, EuiFlexGroup } from '@elastic/eui';
+import React from 'react';
+import { QualityPercentageIndicator } from '../../quality_indicator';
+import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat';
+import { TimeRangeConfig } from '../../../../common/types';
+
+export const FailedDocsPercentageLink = ({
+  isLoading,
+  dataStreamStat,
+  timeRange,
+}: {
+  isLoading: boolean;
+  dataStreamStat: DataStreamStat;
+  timeRange: TimeRangeConfig;
+}) => {
+  const {
+    failedDocs: { percentage },
+  } = dataStreamStat;
+
+  return (
+    <EuiSkeletonRectangle width="50px" height="20px" borderRadius="m" isLoading={isLoading}>
+      <EuiFlexGroup alignItems="center" gutterSize="s">
+        <QualityPercentageIndicator percentage={percentage} />
+      </EuiFlexGroup>
+    </EuiSkeletonRectangle>
+  );
+};