[Dataset quality] Support failure store

packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts

@@ -10,7 +10,11 @@
 import { Client, estypes } from '@elastic/elasticsearch';
 import { pipeline, Readable } from 'stream';
 import { LogDocument } from '@kbn/apm-synthtrace-client/src/lib/logs';
-import { IngestProcessorContainer, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import {
+  IndicesIndexSettings,
+  IngestProcessorContainer,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/types';
 import { ValuesType } from 'utility-types';
 import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client';
 import { getSerializeTransform } from '../shared/get_serialize_transform';
@@ -52,7 +56,11 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
     }
   }
 
-  async createComponentTemplate(name: string, mappings: MappingTypeMapping) {
+  async createComponentTemplate(
+    name: string,
+    mappings?: MappingTypeMapping,
+    settings?: IndicesIndexSettings
+  ) {
     const isTemplateExisting = await this.client.cluster.existsComponentTemplate({ name });
 
     if (isTemplateExisting) return this.logger.info(`Component template already exists: ${name}`);
@@ -61,7 +69,8 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
       await this.client.cluster.putComponentTemplate({
         name,
         template: {
-          mappings,
+          ...((mappings && { mappings }) || {}),
+          ...((settings && { settings }) || {}),
         },
       });
       this.logger.info(`Component template successfully created: ${name}`);
@@ -124,16 +133,17 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
     }
   }
 
-  async createCustomPipeline(processors: IngestProcessorContainer[]) {
+  async createCustomPipeline(processors: IngestProcessorContainer[], pipelineId?: string) {
+    const id = pipelineId ?? LogsCustom;
     try {
       this.client.ingest.putPipeline({
-        id: LogsCustom,
+        id,
         processors,
         version: 1,
       });
-      this.logger.info(`Custom pipeline created: ${LogsCustom}`);
+      this.logger.info(`Custom pipeline created: ${id}`);
     } catch (err) {
-      this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`);
+      this.logger.error(`Custom pipeline creation failed: ${id} - ${err.message}`);
     }
   }
 

x-pack/plugins/data_quality/common/url_schema/common.ts

@@ -9,6 +9,11 @@ import * as rt from 'io-ts';
 
 export const DATA_QUALITY_URL_STATE_KEY = 'pageState';
 
+export const qualityIssuesRT = rt.keyof({
+  degradedDocs: null,
+  failedDocs: null,
+});
+
 export const directionRT = rt.keyof({
   asc: null,
   desc: null,

x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts

@@ -6,7 +6,7 @@
  */
 
 import * as rt from 'io-ts';
-import { dataStreamRT, degradedFieldRT, timeRangeRT } from './common';
+import { dataStreamRT, degradedFieldRT, qualityIssuesRT, timeRangeRT } from './common';
 
 export const urlSchemaRT = rt.exact(
   rt.intersection([
@@ -16,6 +16,7 @@ export const urlSchemaRT = rt.exact(
     rt.partial({
       v: rt.literal(1),
       timeRange: timeRangeRT,
+      qualityIssuesChart: qualityIssuesRT,
       breakdownField: rt.string,
       degradedFields: degradedFieldRT,
       expandedDegradedField: rt.string,

x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts

@@ -17,6 +17,7 @@ export const getStateFromUrlValue = (
     dataStream: urlValue.dataStream,
     timeRange: urlValue.timeRange,
     degradedFields: urlValue.degradedFields,
+    qualityIssuesChart: urlValue.qualityIssuesChart,
     breakdownField: urlValue.breakdownField,
     expandedDegradedField: urlValue.expandedDegradedField,
     showCurrentQualityIssues: urlValue.showCurrentQualityIssues,
@@ -30,6 +31,7 @@ export const getUrlValueFromState = (
     timeRange: state.timeRange,
     degradedFields: state.degradedFields,
     breakdownField: state.breakdownField,
+    qualityIssuesChart: state.qualityIssuesChart,
     expandedDegradedField: state.expandedDegradedField,
     showCurrentQualityIssues: state.showCurrentQualityIssues,
     v: 1,

x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts

@@ -56,6 +56,12 @@ export const getDataStreamDegradedDocsResponseRt = rt.type({
 
 export type DataStreamDegradedDocsResponse = rt.TypeOf<typeof getDataStreamDegradedDocsResponseRt>;
 
+export const getDataStreamFailedDocsResponseRt = rt.type({
+  failedDocs: rt.array(dataStreamDocsStatRt),
+});
+
+export type DataStreamFailedDocsResponse = rt.TypeOf<typeof getDataStreamFailedDocsResponseRt>;
+
 export const integrationDashboardRT = rt.type({
   id: rt.string,
   title: rt.string,
@@ -182,6 +188,7 @@ export type DataStreamSettings = rt.TypeOf<typeof dataStreamSettingsRt>;
 export const dataStreamDetailsRt = rt.partial({
   lastActivity: rt.number,
   degradedDocsCount: rt.number,
+  failedDocsCount: rt.number,
   docsCount: rt.number,
   sizeBytes: rt.number,
   services: rt.record(rt.string, rt.array(rt.string)),

x-pack/plugins/observability_solution/dataset_quality/common/constants.ts

@@ -26,7 +26,7 @@ export const NONE = 'none';
 export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' };
 export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false };
 
-export const DEFAULT_DEGRADED_DOCS = {
+export const DEFAULT_QUALITY_DOC_STATS = {
   count: 0,
   percentage: 0,
 };

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts

@@ -6,12 +6,14 @@
  */
 
 import { DataStreamDocsStat } from '../api_types';
-import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants';
+import { DEFAULT_DATASET_QUALITY, DEFAULT_QUALITY_DOC_STATS } from '../constants';
 import { DataStreamType, QualityIndicators } from '../types';
 import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils';
 import { Integration } from './integration';
 import { DataStreamStatType } from './types';
 
+type QualityStat = Omit<DataStreamDocsStat, 'dataset'> & { percentage: number };
+
 export class DataStreamStat {
   rawName: string;
   type: DataStreamType;
@@ -30,6 +32,10 @@ export class DataStreamStat {
     percentage: number;
     count: number;
   };
+  failedDocs: {
+    percentage: number;
+    count: number;
+  };
 
   private constructor(dataStreamStat: DataStreamStat) {
     this.rawName = dataStreamStat.rawName;
@@ -46,6 +52,7 @@ export class DataStreamStat {
     this.quality = dataStreamStat.quality;
     this.docsInTimeRange = dataStreamStat.docsInTimeRange;
     this.degradedDocs = dataStreamStat.degradedDocs;
+    this.failedDocs = dataStreamStat.failedDocs;
   }
 
   public static create(dataStreamStat: DataStreamStatType) {
@@ -63,36 +70,45 @@ export class DataStreamStat {
       userPrivileges: dataStreamStat.userPrivileges,
       totalDocs: dataStreamStat.totalDocs,
       quality: DEFAULT_DATASET_QUALITY,
-      degradedDocs: DEFAULT_DEGRADED_DOCS,
+      degradedDocs: DEFAULT_QUALITY_DOC_STATS,
+      failedDocs: DEFAULT_QUALITY_DOC_STATS,
     };
 
     return new DataStreamStat(dataStreamStatProps);
   }
 
-  public static fromDegradedDocStat({
+  public static fromQualityStats({
+    datasetName,
     degradedDocStat,
+    failedDocStat,
     datasetIntegrationMap,
     totalDocs,
   }: {
-    degradedDocStat: DataStreamDocsStat & { percentage: number };
+    datasetName: string;
+    degradedDocStat: QualityStat;
+    failedDocStat: QualityStat;
     datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
     totalDocs: number;
   }) {
-    const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset);
+    const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName);
 
     const dataStreamStatProps = {
-      rawName: degradedDocStat.dataset,
+      rawName: datasetName,
       type,
       name: dataset,
       title: datasetIntegrationMap[dataset]?.title || dataset,
       namespace,
       integration: datasetIntegrationMap[dataset]?.integration,
-      quality: mapPercentageToQuality(degradedDocStat.percentage),
+      quality: mapPercentageToQuality([degradedDocStat.percentage, failedDocStat.percentage]),
       docsInTimeRange: totalDocs,
       degradedDocs: {
         percentage: degradedDocStat.percentage,
         count: degradedDocStat.count,
       },
+      failedDocs: {
+        percentage: failedDocStat.percentage,
+        count: failedDocStat.count,
+      },
     };
 
     return new DataStreamStat(dataStreamStatProps);
@@ -102,8 +118,4 @@ export class DataStreamStat {
     const avgDocSize = sizeBytes && totalDocs ? sizeBytes / totalDocs : 0;
     return avgDocSize * (docsInTimeRange ?? 0);
   }
-
-  public static calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) {
-    return totalDocs && count ? (count / totalDocs) * 100 : 0;
-  }
 }

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts

@@ -18,6 +18,9 @@ export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse;
 export type GetDataStreamsDegradedDocsStatsParams =
   APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params'];
 export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query'];
+export type GetDataStreamsFailedDocsStatsParams =
+  APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/failed_docs`>['params'];
+export type GetDataStreamsFailedDocsStatsQuery = GetDataStreamsFailedDocsStatsParams['query'];
 
 /*
 Types for stats based in documents inside a DataStream

x-pack/plugins/observability_solution/dataset_quality/common/translations.ts

@@ -93,12 +93,9 @@ export const flyoutSummaryText = i18n.translate('xpack.datasetQuality.flyoutSumm
   defaultMessage: 'Summary',
 });
 
-export const overviewDegradedDocsText = i18n.translate(
-  'xpack.datasetQuality.flyout.degradedDocsTitle',
-  {
-    defaultMessage: 'Degraded docs',
-  }
-);
+export const overviewTrendsDocsText = i18n.translate('xpack.datasetQuality.flyout.trendDocsTitle', {
+  defaultMessage: 'Document trends',
+});
 
 export const flyoutDegradedDocsTrendText = i18n.translate(
   'xpack.datasetQuality.flyoutDegradedDocsViz',
@@ -107,6 +104,13 @@ export const flyoutDegradedDocsTrendText = i18n.translate(
   }
 );
 
+export const flyoutFailedDocsTrendText = i18n.translate(
+  'xpack.datasetQuality.flyoutFailedDocsViz',
+  {
+    defaultMessage: 'Failed documents trend',
+  }
+);
+
 export const flyoutDegradedDocsPercentageText = i18n.translate(
   'xpack.datasetQuality.flyoutDegradedDocsPercentage',
   {
@@ -115,6 +119,14 @@ export const flyoutDegradedDocsPercentageText = i18n.translate(
   }
 );
 
+export const flyoutFailedDocsPercentageText = i18n.translate(
+  'xpack.datasetQuality.flyoutFailedDocsPercentage',
+  {
+    defaultMessage: 'Failed docs %',
+    description: 'Tooltip label for the percentage of failed documents chart.',
+  }
+);
+
 export const flyoutDocsCountTotalText = i18n.translate(
   'xpack.datasetQuality.flyoutDocsCountTotal',
   {
@@ -151,14 +163,14 @@ export const summaryPanelLast24hText = i18n.translate(
 export const summaryPanelQualityText = i18n.translate(
   'xpack.datasetQuality.summaryPanelQualityText',
   {
-    defaultMessage: 'Data Sets Quality',
+    defaultMessage: 'Data Set Quality',
   }
 );
 
 export const summaryPanelQualityTooltipText = i18n.translate(
   'xpack.datasetQuality.summaryPanelQualityTooltipText',
   {
-    defaultMessage: 'Quality is based on the percentage of degraded docs in a data set.',
+    defaultMessage: 'Quality is based on the percentage of degraded and failed docs in a data set.',
   }
 );
 
@@ -319,6 +331,13 @@ export const overviewPanelDatasetQualityIndicatorDegradedDocs = i18n.translate(
   }
 );
 
+export const overviewPanelDatasetQualityIndicatorFailedDocs = i18n.translate(
+  'xpack.datasetQuality.details.overviewPanel.datasetQuality.failedDocs',
+  {
+    defaultMessage: 'Failed docs',
+  }
+);
+
 export const overviewDegradedFieldsTableLoadingText = i18n.translate(
   'xpack.datasetQuality.details.degradedFieldsTableLoadingText',
   {

x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts

@@ -84,5 +84,11 @@ describe('dataset_name', () => {
         extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001')
       ).toEqual('metrics-apm.app.adservice-default');
     });
+
+    it('returns the correct index name if backing index is a failure store index', () => {
+      expect(
+        extractIndexNameFromBackingIndex('.fs-logs-elastic_agent-default-2024.11.11-000001')
+      ).toEqual('logs-elastic_agent-default');
+    });
   });
 });

x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts

@@ -40,7 +40,8 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => {
 };
 
 export const extractIndexNameFromBackingIndex = (indexString: string): string => {
-  const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
+  // TODO: Undo this change once ::failures is supported
+  const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
   const match = indexString.match(pattern);
 
   return match ? match[1] : indexString;

x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts

@@ -8,10 +8,14 @@
 import { POOR_QUALITY_MINIMUM_PERCENTAGE, DEGRADED_QUALITY_MINIMUM_PERCENTAGE } from '../constants';
 import { QualityIndicators } from '../types';
 
-export const mapPercentageToQuality = (percentage: number): QualityIndicators => {
-  return percentage > POOR_QUALITY_MINIMUM_PERCENTAGE
-    ? 'poor'
-    : percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE
-    ? 'degraded'
-    : 'good';
+export const mapPercentageToQuality = (percentages: number[]): QualityIndicators => {
+  if (percentages.some((percentage) => percentage > POOR_QUALITY_MINIMUM_PERCENTAGE)) {
+    return 'poor';
+  }
+
+  if (percentages.some((percentage) => percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE)) {
+    return 'degraded';
+  }
+
+  return 'good';
 };