diff --git a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts index 3e95383d9dbb9..6f1f1ca4ec6af 100644 --- a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts @@ -10,7 +10,11 @@ import { Client, estypes } from '@elastic/elasticsearch'; import { pipeline, Readable } from 'stream'; import { LogDocument } from '@kbn/apm-synthtrace-client/src/lib/logs'; -import { IngestProcessorContainer, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { + IndicesIndexSettings, + IngestProcessorContainer, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/types'; import { ValuesType } from 'utility-types'; import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; import { getSerializeTransform } from '../shared/get_serialize_transform'; @@ -52,7 +56,11 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { } } - async createComponentTemplate(name: string, mappings: MappingTypeMapping) { + async createComponentTemplate( + name: string, + mappings?: MappingTypeMapping, + settings?: IndicesIndexSettings + ) { const isTemplateExisting = await this.client.cluster.existsComponentTemplate({ name }); if (isTemplateExisting) return this.logger.info(`Component template already exists: ${name}`); @@ -61,7 +69,8 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { await this.client.cluster.putComponentTemplate({ name, template: { - mappings, + ...((mappings && { mappings }) || {}), + ...((settings && { settings }) || {}), }, }); this.logger.info(`Component template successfully created: ${name}`); @@ -124,16 +133,17 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { } } - async createCustomPipeline(processors: IngestProcessorContainer[]) { + async createCustomPipeline(processors: IngestProcessorContainer[], pipelineId?: string) { + const id = pipelineId ?? LogsCustom; try { this.client.ingest.putPipeline({ - id: LogsCustom, + id, processors, version: 1, }); - this.logger.info(`Custom pipeline created: ${LogsCustom}`); + this.logger.info(`Custom pipeline created: ${id}`); } catch (err) { - this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`); + this.logger.error(`Custom pipeline creation failed: ${id} - ${err.message}`); } } diff --git a/x-pack/plugins/data_quality/common/url_schema/common.ts b/x-pack/plugins/data_quality/common/url_schema/common.ts index eb929faee1b00..a60488568d1a8 100644 --- a/x-pack/plugins/data_quality/common/url_schema/common.ts +++ b/x-pack/plugins/data_quality/common/url_schema/common.ts @@ -9,6 +9,11 @@ import * as rt from 'io-ts'; export const DATA_QUALITY_URL_STATE_KEY = 'pageState'; +export const qualityIssuesRT = rt.keyof({ + degraded: null, + failed: null, +}); + export const directionRT = rt.keyof({ asc: null, desc: null, diff --git a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts index 97c7771bbf994..deba69f697245 100644 --- a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts +++ b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts @@ -6,7 +6,7 @@ */ import * as rt from 'io-ts'; -import { dataStreamRT, degradedFieldRT, timeRangeRT } from './common'; +import { dataStreamRT, degradedFieldRT, qualityIssuesRT, timeRangeRT } from './common'; export const urlSchemaRT = rt.exact( rt.intersection([ @@ -16,6 +16,7 @@ export const urlSchemaRT = rt.exact( rt.partial({ v: rt.literal(1), timeRange: timeRangeRT, + qualityIssuesChart: qualityIssuesRT, breakdownField: rt.string, degradedFields: degradedFieldRT, expandedDegradedField: rt.string, diff --git a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v2.ts b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v2.ts new file mode 100644 index 0000000000000..1116ad304aebd --- /dev/null +++ b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v2.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as rt from 'io-ts'; +import { dataStreamRT, degradedFieldRT, qualityIssuesRT, timeRangeRT } from './common'; + +export const urlSchemaRT = rt.exact( + rt.intersection([ + rt.type({ + dataStream: dataStreamRT, + }), + rt.partial({ + v: rt.literal(2), + timeRange: timeRangeRT, + qualityIssuesChart: qualityIssuesRT, + breakdownField: rt.string, + degradedFields: degradedFieldRT, + expandedQualityIssue: rt.type({ + name: rt.string, + type: qualityIssuesRT, + }), + showCurrentQualityIssues: rt.boolean, + }), + ]) +); + +export type UrlSchema = rt.TypeOf; diff --git a/x-pack/plugins/data_quality/common/url_schema/index.ts b/x-pack/plugins/data_quality/common/url_schema/index.ts index bf984fa73f129..912d96ad05d72 100644 --- a/x-pack/plugins/data_quality/common/url_schema/index.ts +++ b/x-pack/plugins/data_quality/common/url_schema/index.ts @@ -8,3 +8,4 @@ export { DATA_QUALITY_URL_STATE_KEY } from './common'; export * as datasetQualityUrlSchemaV1 from './dataset_quality_url_schema_v1'; export * as datasetQualityDetailsUrlSchemaV1 from './dataset_quality_details_url_schema_v1'; +export * as datasetQualityDetailsUrlSchemaV2 from './dataset_quality_details_url_schema_v2'; diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts index 7b91895598eca..c648686715515 100644 --- a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts +++ b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts @@ -17,8 +17,14 @@ export const getStateFromUrlValue = ( dataStream: urlValue.dataStream, timeRange: urlValue.timeRange, degradedFields: urlValue.degradedFields, + qualityIssuesChart: urlValue.qualityIssuesChart, breakdownField: urlValue.breakdownField, - expandedDegradedField: urlValue.expandedDegradedField, + expandedQualityIssue: urlValue.expandedDegradedField + ? { + name: urlValue.expandedDegradedField, + type: 'degraded', + } + : undefined, showCurrentQualityIssues: urlValue.showCurrentQualityIssues, }); @@ -30,7 +36,8 @@ export const getUrlValueFromState = ( timeRange: state.timeRange, degradedFields: state.degradedFields, breakdownField: state.breakdownField, - expandedDegradedField: state.expandedDegradedField, + qualityIssuesChart: state.qualityIssuesChart, + expandedDegradedField: state.expandedQualityIssue?.name, showCurrentQualityIssues: state.showCurrentQualityIssues, v: 1, }); diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v2.ts b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v2.ts new file mode 100644 index 0000000000000..339d5a04077d2 --- /dev/null +++ b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v2.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DatasetQualityDetailsPublicStateUpdate } from '@kbn/dataset-quality-plugin/public/controller/dataset_quality_details'; +import * as rt from 'io-ts'; +import { deepCompactObject } from '../../../common/utils/deep_compact_object'; +import { datasetQualityDetailsUrlSchemaV2 } from '../../../common/url_schema'; + +export const getStateFromUrlValue = ( + urlValue: datasetQualityDetailsUrlSchemaV2.UrlSchema +): DatasetQualityDetailsPublicStateUpdate => + deepCompactObject({ + dataStream: urlValue.dataStream, + timeRange: urlValue.timeRange, + degradedFields: urlValue.degradedFields, + qualityIssuesChart: urlValue.qualityIssuesChart, + breakdownField: urlValue.breakdownField, + expandedQualityIssue: urlValue.expandedQualityIssue, + showCurrentQualityIssues: urlValue.showCurrentQualityIssues, + }); + +export const getUrlValueFromState = ( + state: DatasetQualityDetailsPublicStateUpdate +): datasetQualityDetailsUrlSchemaV2.UrlSchema => + deepCompactObject({ + dataStream: state.dataStream, + timeRange: state.timeRange, + degradedFields: state.degradedFields, + breakdownField: state.breakdownField, + qualityIssuesChart: state.qualityIssuesChart, + expandedQualityIssue: state.expandedQualityIssue, + showCurrentQualityIssues: state.showCurrentQualityIssues, + v: 2, + }); + +const stateFromUrlSchemaRT = new rt.Type< + DatasetQualityDetailsPublicStateUpdate, + datasetQualityDetailsUrlSchemaV2.UrlSchema, + datasetQualityDetailsUrlSchemaV2.UrlSchema +>( + 'stateFromUrlSchemaRT', + rt.never.is, + (urlSchema, _context) => rt.success(getStateFromUrlValue(urlSchema)), + getUrlValueFromState +); + +export const stateFromUntrustedUrlRT = + datasetQualityDetailsUrlSchemaV2.urlSchemaRT.pipe(stateFromUrlSchemaRT); diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_state_storage_service.ts b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_state_storage_service.ts index 1a71ee6cc33ed..6dfd8c83b2bf0 100644 --- a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_state_storage_service.ts +++ b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_state_storage_service.ts @@ -14,6 +14,7 @@ import { DatasetQualityDetailsPublicStateUpdate } from '@kbn/dataset-quality-plu import * as rt from 'io-ts'; import { DATA_QUALITY_URL_STATE_KEY } from '../../../common/url_schema'; import * as urlSchemaV1 from './url_schema_v1'; +import * as urlSchemaV2 from './url_schema_v2'; export const updateUrlFromDatasetQualityDetailsState = ({ urlStateStorageContainer, @@ -26,7 +27,8 @@ export const updateUrlFromDatasetQualityDetailsState = ({ return; } - const encodedUrlStateValues = urlSchemaV1.stateFromUntrustedUrlRT.encode( + // we want to use always the newest schema + const encodedUrlStateValues = urlSchemaV2.stateFromUntrustedUrlRT.encode( datasetQualityDetailsState ); @@ -50,7 +52,7 @@ export const getDatasetQualityDetailsStateFromUrl = ({ urlStateStorageContainer.get(DATA_QUALITY_URL_STATE_KEY) ?? undefined; const stateValuesE = rt - .union([rt.undefined, urlSchemaV1.stateFromUntrustedUrlRT]) + .union([rt.undefined, urlSchemaV1.stateFromUntrustedUrlRT, urlSchemaV2.stateFromUntrustedUrlRT]) .decode(urlStateValues); if (Either.isLeft(stateValuesE)) { diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index 51a1421aec918..1c1cf8730ff45 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -56,6 +56,12 @@ export const getDataStreamDegradedDocsResponseRt = rt.type({ export type DataStreamDegradedDocsResponse = rt.TypeOf; +export const getDataStreamFailedDocsResponseRt = rt.type({ + failedDocs: rt.array(dataStreamDocsStatRt), +}); + +export type DataStreamFailedDocsResponse = rt.TypeOf; + export const integrationDashboardRT = rt.type({ id: rt.string, title: rt.string, @@ -103,19 +109,55 @@ export const getIntegrationsResponseRt = rt.exact( export type IntegrationResponse = rt.TypeOf; -export const degradedFieldRt = rt.type({ - name: rt.string, +export const qualityIssueBaseRT = rt.type({ count: rt.number, - lastOccurrence: rt.union([rt.null, rt.number]), + lastOccurrence: rt.union([rt.undefined, rt.null, rt.number]), timeSeries: rt.array( rt.type({ x: rt.number, y: rt.number, }) ), - indexFieldWasLastPresentIn: rt.string, }); +export const qualityIssueRT = rt.intersection([ + qualityIssueBaseRT, + rt.partial({ + name: rt.string, + indexFieldWasLastPresentIn: rt.string, + }), + rt.type({ + name: rt.string, + type: rt.keyof({ + degraded: null, + failed: null, + }), + }), +]); + +export type QualityIssue = rt.TypeOf; + +export type FailedDocsDetails = rt.TypeOf; + +export const failedDocsErrorRt = rt.type({ + message: rt.string, + type: rt.string, +}); + +export const failedDocsErrorsRt = rt.type({ + errors: rt.array(failedDocsErrorRt), +}); + +export type FailedDocsErrors = rt.TypeOf; + +export const degradedFieldRt = rt.intersection([ + qualityIssueBaseRT, + rt.type({ + name: rt.string, + indexFieldWasLastPresentIn: rt.string, + }), +]); + export type DegradedField = rt.TypeOf; export const getDataStreamDegradedFieldsResponseRt = rt.type({ @@ -182,6 +224,7 @@ export type DataStreamSettings = rt.TypeOf; export const dataStreamDetailsRt = rt.partial({ lastActivity: rt.number, degradedDocsCount: rt.number, + failedDocsCount: rt.number, docsCount: rt.number, sizeBytes: rt.number, services: rt.record(rt.string, rt.array(rt.string)), diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index 74809e0e19420..fca2bcc83f3ab 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -26,7 +26,7 @@ export const NONE = 'none'; export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' }; export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false }; -export const DEFAULT_DEGRADED_DOCS = { +export const DEFAULT_QUALITY_DOC_STATS = { count: 0, percentage: 0, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 094d92ff3fea6..d9673d0788921 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -6,12 +6,14 @@ */ import { DataStreamDocsStat } from '../api_types'; -import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants'; +import { DEFAULT_DATASET_QUALITY, DEFAULT_QUALITY_DOC_STATS } from '../constants'; import { DataStreamType, QualityIndicators } from '../types'; import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils'; import { Integration } from './integration'; import { DataStreamStatType } from './types'; +type QualityStat = Omit & { percentage: number }; + export class DataStreamStat { rawName: string; type: DataStreamType; @@ -30,6 +32,10 @@ export class DataStreamStat { percentage: number; count: number; }; + failedDocs: { + percentage: number; + count: number; + }; private constructor(dataStreamStat: DataStreamStat) { this.rawName = dataStreamStat.rawName; @@ -46,6 +52,7 @@ export class DataStreamStat { this.quality = dataStreamStat.quality; this.docsInTimeRange = dataStreamStat.docsInTimeRange; this.degradedDocs = dataStreamStat.degradedDocs; + this.failedDocs = dataStreamStat.failedDocs; } public static create(dataStreamStat: DataStreamStatType) { @@ -63,36 +70,45 @@ export class DataStreamStat { userPrivileges: dataStreamStat.userPrivileges, totalDocs: dataStreamStat.totalDocs, quality: DEFAULT_DATASET_QUALITY, - degradedDocs: DEFAULT_DEGRADED_DOCS, + degradedDocs: DEFAULT_QUALITY_DOC_STATS, + failedDocs: DEFAULT_QUALITY_DOC_STATS, }; return new DataStreamStat(dataStreamStatProps); } - public static fromDegradedDocStat({ + public static fromQualityStats({ + datasetName, degradedDocStat, + failedDocStat, datasetIntegrationMap, totalDocs, }: { - degradedDocStat: DataStreamDocsStat & { percentage: number }; + datasetName: string; + degradedDocStat: QualityStat; + failedDocStat: QualityStat; datasetIntegrationMap: Record; totalDocs: number; }) { - const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset); + const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName); const dataStreamStatProps = { - rawName: degradedDocStat.dataset, + rawName: datasetName, type, name: dataset, title: datasetIntegrationMap[dataset]?.title || dataset, namespace, integration: datasetIntegrationMap[dataset]?.integration, - quality: mapPercentageToQuality(degradedDocStat.percentage), + quality: mapPercentageToQuality([degradedDocStat.percentage, failedDocStat.percentage]), docsInTimeRange: totalDocs, degradedDocs: { percentage: degradedDocStat.percentage, count: degradedDocStat.count, }, + failedDocs: { + percentage: failedDocStat.percentage, + count: failedDocStat.count, + }, }; return new DataStreamStat(dataStreamStatProps); @@ -102,8 +118,4 @@ export class DataStreamStat { const avgDocSize = sizeBytes && totalDocs ? sizeBytes / totalDocs : 0; return avgDocSize * (docsInTimeRange ?? 0); } - - public static calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) { - return totalDocs && count ? (count / totalDocs) * 100 : 0; - } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index bc0c12d234d26..bbc017e846b50 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -14,7 +14,6 @@ export type GetDataStreamsStatsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>; export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0]; export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse; - export type GetDataStreamsDegradedDocsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params']; export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query']; @@ -80,3 +79,20 @@ export type { DegradedField, DegradedFieldResponse, } from '../api_types'; + +/* + Types for Failure store information +*/ +export type GetDataStreamsFailedDocsStatsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/failed_docs`>['params']; +export type GetDataStreamsFailedDocsStatsQuery = GetDataStreamsFailedDocsStatsParams['query']; +export type GetDataStreamsFailedDocsDetailsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs`>['params']; +export type GetDataStreamsFailedDocsDetailsQuery = GetDataStreamsFailedDocsDetailsParams['query']; +export type GetDataStreamFailedDocsDetailsParams = GetDataStreamsFailedDocsDetailsParams['path'] & + GetDataStreamsFailedDocsDetailsQuery; +export type GetDataStreamsFailedDocsErrorsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs/errors`>['params']; +export type GetDataStreamsFailedDocsErrorsQuery = GetDataStreamsFailedDocsErrorsParams['query']; +export type GetDataStreamFailedDocsErrorsParams = GetDataStreamsFailedDocsErrorsParams['path'] & + GetDataStreamsFailedDocsErrorsQuery; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 1026dd8ea58d3..96e3605f2a3c3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -93,12 +93,9 @@ export const flyoutSummaryText = i18n.translate('xpack.datasetQuality.flyoutSumm defaultMessage: 'Summary', }); -export const overviewDegradedDocsText = i18n.translate( - 'xpack.datasetQuality.flyout.degradedDocsTitle', - { - defaultMessage: 'Degraded docs', - } -); +export const overviewTrendsDocsText = i18n.translate('xpack.datasetQuality.flyout.trendDocsTitle', { + defaultMessage: 'Document trends', +}); export const flyoutDegradedDocsTrendText = i18n.translate( 'xpack.datasetQuality.flyoutDegradedDocsViz', @@ -107,6 +104,13 @@ export const flyoutDegradedDocsTrendText = i18n.translate( } ); +export const flyoutFailedDocsTrendText = i18n.translate( + 'xpack.datasetQuality.flyoutFailedDocsViz', + { + defaultMessage: 'Failed documents trend', + } +); + export const flyoutDegradedDocsPercentageText = i18n.translate( 'xpack.datasetQuality.flyoutDegradedDocsPercentage', { @@ -115,6 +119,14 @@ export const flyoutDegradedDocsPercentageText = i18n.translate( } ); +export const flyoutFailedDocsPercentageText = i18n.translate( + 'xpack.datasetQuality.flyoutFailedDocsPercentage', + { + defaultMessage: 'Failed docs %', + description: 'Tooltip label for the percentage of failed documents chart.', + } +); + export const flyoutDocsCountTotalText = i18n.translate( 'xpack.datasetQuality.flyoutDocsCountTotal', { @@ -151,14 +163,14 @@ export const summaryPanelLast24hText = i18n.translate( export const summaryPanelQualityText = i18n.translate( 'xpack.datasetQuality.summaryPanelQualityText', { - defaultMessage: 'Data Sets Quality', + defaultMessage: 'Data Set Quality', } ); export const summaryPanelQualityTooltipText = i18n.translate( 'xpack.datasetQuality.summaryPanelQualityTooltipText', { - defaultMessage: 'Quality is based on the percentage of degraded docs in a data set.', + defaultMessage: 'Quality is based on the percentage of degraded and failed docs in a data set.', } ); @@ -319,6 +331,13 @@ export const overviewPanelDatasetQualityIndicatorDegradedDocs = i18n.translate( } ); +export const overviewPanelDatasetQualityIndicatorFailedDocs = i18n.translate( + 'xpack.datasetQuality.details.overviewPanel.datasetQuality.failedDocs', + { + defaultMessage: 'Failed docs', + } +); + export const overviewDegradedFieldsTableLoadingText = i18n.translate( 'xpack.datasetQuality.details.degradedFieldsTableLoadingText', { @@ -326,10 +345,10 @@ export const overviewDegradedFieldsTableLoadingText = i18n.translate( } ); -export const overviewDegradedFieldsTableNoData = i18n.translate( - 'xpack.datasetQuality.details.degradedFieldsTableNoData', +export const qualityIssuesTableNoData = i18n.translate( + 'xpack.datasetQuality.details.qualityIssuesTableNoData', { - defaultMessage: 'No degraded fields found', + defaultMessage: 'No quality issues found', } ); @@ -400,21 +419,31 @@ export const integrationVersionText = i18n.translate( defaultMessage: 'Version', } ); -export const fieldColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.field', { - defaultMessage: 'Field', +export const issueColumnName = i18n.translate('xpack.datasetQuality.details.qualityIssues.issue', { + defaultMessage: 'Issue', }); -export const countColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.count', { - defaultMessage: 'Docs count', -}); +export const countColumnName = i18n.translate( + 'xpack.datasetQuality.details.qualityIssues.docsCount', + { + defaultMessage: 'Docs count', + } +); export const lastOccurrenceColumnName = i18n.translate( - 'xpack.datasetQuality.details.degradedField.lastOccurrence', + 'xpack.datasetQuality.details.qualityIssues.lastOccurrence', { defaultMessage: 'Last occurrence', } ); +export const documentIndexFailed = i18n.translate( + 'xpack.datasetQuality.details.qualityIssues.documentIndexFailed', + { + defaultMessage: 'Documents indexing failed', + } +); + export const degradedFieldValuesColumnName = i18n.translate( 'xpack.datasetQuality.details.degradedField.values', { @@ -679,3 +708,10 @@ export const manualMitigationCustomPipelineCreateEditPipelineLink = i18n.transla defaultMessage: 'create or edit the pipeline', } ); + +export const failedDocsErrorsColumnName = i18n.translate( + 'xpack.datasetQuality.details.failedDocs.errors', + { + defaultMessage: 'Error messages', + } +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts index 8ffcbfe5657fa..ad6d2678932b8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts @@ -84,5 +84,11 @@ describe('dataset_name', () => { extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001') ).toEqual('metrics-apm.app.adservice-default'); }); + + it('returns the correct index name if backing index is a failure store index', () => { + expect( + extractIndexNameFromBackingIndex('.fs-logs-elastic_agent-default-2024.11.11-000001') + ).toEqual('logs-elastic_agent-default'); + }); }); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts index eaca58ded6404..5cf347ed09304 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts @@ -40,7 +40,8 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => { }; export const extractIndexNameFromBackingIndex = (indexString: string): string => { - const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; + // TODO: Undo this change once ::failures is supported + const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; const match = indexString.match(pattern); return match ? match[1] : indexString; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts index 62e0411e541b0..45dcbdd19ff9f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts @@ -8,10 +8,14 @@ import { POOR_QUALITY_MINIMUM_PERCENTAGE, DEGRADED_QUALITY_MINIMUM_PERCENTAGE } from '../constants'; import { QualityIndicators } from '../types'; -export const mapPercentageToQuality = (percentage: number): QualityIndicators => { - return percentage > POOR_QUALITY_MINIMUM_PERCENTAGE - ? 'poor' - : percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE - ? 'degraded' - : 'good'; +export const mapPercentageToQuality = (percentages: number[]): QualityIndicators => { + if (percentages.some((percentage) => percentage > POOR_QUALITY_MINIMUM_PERCENTAGE)) { + return 'poor'; + } + + if (percentages.some((percentage) => percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE)) { + return 'degraded'; + } + + return 'good'; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx index b186c16c0c0f8..36cd65423d83b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx @@ -29,20 +29,11 @@ import { summaryPanelQualityText, summaryPanelQualityTooltipText, } from '../../../../common/translations'; -import { mapPercentagesToQualityCounts } from '../../quality_indicator'; export function DatasetsQualityIndicators() { const { onPageReady } = usePerformanceContext(); - const { - datasetsQuality, - isDatasetsQualityLoading, - datasetsActivity, - numberOfDatasets, - numberOfDocuments, - } = useSummaryPanelContext(); - const qualityCounts = mapPercentagesToQualityCounts(datasetsQuality.percentages); - const datasetsWithoutIgnoredField = - datasetsActivity.total > 0 ? datasetsActivity.total - datasetsQuality.percentages.length : 0; + const { datasetsQuality, isDatasetsQualityLoading, numberOfDatasets, numberOfDocuments } = + useSummaryPanelContext(); if (!isDatasetsQualityLoading && (numberOfDatasets || numberOfDocuments)) { onPageReady({ @@ -66,21 +57,21 @@ export function DatasetsQualityIndicators() { ); +const failedDocsColumnTooltip = ( + +); + const datasetQualityColumnTooltip = ( @@ -159,7 +171,9 @@ export const getDatasetQualityTableColumns = ({ canUserMonitorDataset, canUserMonitorAnyDataStream, loadingDataStreamStats, + loadingDocStats, loadingDegradedStats, + loadingFailedStats, showFullDatasetNames, isActiveDataset, timeRange, @@ -169,7 +183,9 @@ export const getDatasetQualityTableColumns = ({ canUserMonitorDataset: boolean; canUserMonitorAnyDataStream: boolean; loadingDataStreamStats: boolean; + loadingDocStats: boolean; loadingDegradedStats: boolean; + loadingFailedStats: boolean; showFullDatasetNames: boolean; isActiveDataset: (lastActivity: number) => boolean; timeRange: TimeRangeConfig; @@ -248,7 +264,7 @@ export const getDatasetQualityTableColumns = ({ width="60px" height="20px" borderRadius="m" - isLoading={loadingDataStreamStats || loadingDegradedStats} + isLoading={loadingDataStreamStats || loadingDocStats} > {formatNumber( DataStreamStat.calculateFilteredSize(dataStreamStat), @@ -273,11 +289,11 @@ export const getDatasetQualityTableColumns = ({ ), - field: 'degradedDocs.percentage', + field: 'quality', sortable: true, render: (_, dataStreamStat: DataStreamStat) => ( ), @@ -307,35 +323,61 @@ export const getDatasetQualityTableColumns = ({ }, { name: ( - - {lastActivityColumnName} + + + + {`${failedDocsColumnName} `} + + + ), - field: 'lastActivity', - render: (timestamp: number) => ( - - {!isActiveDataset(timestamp) ? ( - - {inactiveDatasetActivityColumnDescription} - - - - - ) : ( - fieldFormats - .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) - .convert(timestamp) - )} - - ), - width: '300px', + field: 'failedDocs.percentage', sortable: true, + render: (_, dataStreamStat: DataStreamStat) => ( + + ), + width: '140px', }, + ...(canUserMonitorDataset && canUserMonitorAnyDataStream + ? [ + { + name: ( + + {lastActivityColumnName} + + ), + field: 'lastActivity', + render: (timestamp: number) => ( + + {!isActiveDataset(timestamp) ? ( + + {inactiveDatasetActivityColumnDescription} + + + + + ) : ( + fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) + .convert(timestamp) + )} + + ), + width: '300px', + sortable: true, + }, + ] + : []), { name: actionsColumnName, render: (dataStreamStat: DataStreamStat) => ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx index 9d32c84891a34..6097736a857de 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx @@ -7,6 +7,7 @@ import { EuiSkeletonRectangle, EuiFlexGroup, EuiLink } from '@elastic/eui'; import React from 'react'; +import { i18n } from '@kbn/i18n'; import { _IGNORED } from '../../../../common/es_fields'; import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks'; import { QualityPercentageIndicator } from '../../quality_indicator'; @@ -38,6 +39,14 @@ export const DegradedDocsPercentageLink = ({ timeRangeConfig: timeRange, }); + const tooltip = (degradedDocsCount: number) => + i18n.translate('xpack.datasetQuality.fewDegradedDocsTooltip', { + defaultMessage: '{degradedDocsCount} degraded docs in this data set.', + values: { + degradedDocsCount, + }, + }); + return ( @@ -46,10 +55,14 @@ export const DegradedDocsPercentageLink = ({ data-test-subj="datasetQualityDegradedDocsPercentageLink" {...redirectLinkProps.linkProps} > - + ) : ( - + )} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx new file mode 100644 index 0000000000000..2ba6b22b63771 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiSkeletonRectangle, EuiFlexGroup, EuiLink } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks'; +import { QualityPercentageIndicator } from '../../quality_indicator'; +import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; +import { TimeRangeConfig } from '../../../../common/types'; + +export const FailedDocsPercentageLink = ({ + isLoading, + dataStreamStat, + timeRange, +}: { + isLoading: boolean; + dataStreamStat: DataStreamStat; + timeRange: TimeRangeConfig; +}) => { + const { + failedDocs: { percentage, count }, + } = dataStreamStat; + + const { sendTelemetry } = useDatasetRedirectLinkTelemetry({ + rawName: dataStreamStat.rawName, + query: { language: 'kuery', query: '' }, + }); + + const redirectLinkProps = useRedirectLink({ + dataStreamStat, + query: { language: 'kuery', query: '' }, + sendTelemetry, + timeRangeConfig: timeRange, + }); + + const tooltip = (failedDocsCount: number) => + i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', { + defaultMessage: '{failedDocsCount} failed docs in this data set.', + values: { + failedDocsCount, + }, + }); + + return ( + + + {percentage ? ( + + + + ) : ( + + )} + + + ); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx index 59a1ae3d39d62..427952027f162 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/dataset_quality_details.tsx @@ -13,7 +13,7 @@ import { Header } from './header'; import { Overview } from './overview'; import { Details } from './details'; -const DegradedFieldFlyout = dynamic(() => import('./degraded_field_flyout')); +const DegradedFieldFlyout = dynamic(() => import('./quality_issue_flyout')); // Allow for lazy loading // eslint-disable-next-line import/no-default-export diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/columns.tsx index 30c67ddeb9c34..3f489db6e9630 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/columns.tsx @@ -8,16 +8,19 @@ import React from 'react'; import { css } from '@emotion/react'; import { i18n } from '@kbn/i18n'; -import { EuiBasicTableColumn, EuiButtonIcon } from '@elastic/eui'; +import { EuiBasicTableColumn, EuiButtonIcon, EuiText } from '@elastic/eui'; import type { FieldFormat } from '@kbn/field-formats-plugin/common'; import { formatNumber } from '@elastic/eui'; -import { DegradedField } from '../../../../../common/api_types'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { QualityIssueType } from '../../../../state_machines/dataset_quality_details_controller'; +import { QualityIssue } from '../../../../../common/api_types'; import { SparkPlot } from '../../../common/spark_plot'; import { NUMBER_FORMAT } from '../../../../../common/constants'; import { countColumnName, - fieldColumnName, + documentIndexFailed, + issueColumnName, lastOccurrenceColumnName, } from '../../../../../common/translations'; @@ -42,17 +45,21 @@ export const getDegradedFieldsColumns = ({ }: { dateFormatter: FieldFormat; isLoading: boolean; - expandedDegradedField?: string; - openDegradedFieldFlyout: (name: string) => void; -}): Array> => [ + expandedDegradedField?: { + name: string; + type: QualityIssueType; + }; + openDegradedFieldFlyout: (name: string, type: QualityIssueType) => void; +}): Array> => [ { name: '', field: 'name', - render: (_, { name }) => { - const isExpanded = name === expandedDegradedField; + render: (_, { name, type }) => { + const isExpanded = + name === expandedDegradedField?.name && type === expandedDegradedField?.type; const onExpandClick = () => { - openDegradedFieldFlyout(name); + openDegradedFieldFlyout(name, type); }; return ( @@ -75,8 +82,27 @@ export const getDegradedFieldsColumns = ({ `, }, { - name: fieldColumnName, + name: issueColumnName, field: 'name', + render: (_, { name, type }) => { + return type === 'degraded' ? ( + + + {name}{' '} + + ), + }} + /> + + ) : ( + <>{documentIndexFailed} + ); + }, }, { name: countColumnName, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx index 0cdc460cd56dc..7239890ef6fc0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/degraded_fields.tsx @@ -33,7 +33,7 @@ export function DegradedFields() { }); const toggleTextSwitchId = useGeneratedHtmlId({ prefix: 'toggleTextSwitch' }); - const { totalItemCount, toggleCurrentQualityIssues, showCurrentQualityIssues } = + const { totalItemCount, toggleCurrentQualityIssues, showCurrentQualityIssues, renderedItems } = useDegradedFields(); const latestBackingIndexToggle = ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/table.tsx index 14f0227e57c19..b993e0ef21da3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/degraded_fields/table.tsx @@ -11,7 +11,7 @@ import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import { getDegradedFieldsColumns } from './columns'; import { overviewDegradedFieldsTableLoadingText, - overviewDegradedFieldsTableNoData, + qualityIssuesTableNoData, } from '../../../../../common/translations'; import { useDegradedFields } from '../../../../hooks/use_degraded_fields'; @@ -56,7 +56,7 @@ export const DegradedFieldTable = () => { {overviewDegradedFieldsTableNoData}} + title={

{qualityIssuesTableNoData}

} hasBorder={false} titleSize="m" /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts new file mode 100644 index 0000000000000..4e76999318f57 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public'; +import { v4 as uuidv4 } from 'uuid'; + +import { + flyoutFailedDocsTrendText, + flyoutFailedDocsPercentageText, +} from '../../../../../../common/translations'; + +enum DatasetQualityLensColumn { + Date = 'date_column', + FailedDocs = 'failed_docs_column', + CountFailed = 'count_failed', + CountTotal = 'count_total', + Math = 'math_column', + Breakdown = 'breakdown_column', +} + +const MAX_BREAKDOWN_SERIES = 5; + +interface GetLensAttributesParams { + color: string; + dataStream: string; + datasetTitle: string; + breakdownFieldName?: string; +} + +export function getLensAttributes({ + color, + dataStream, + datasetTitle, + breakdownFieldName, +}: GetLensAttributesParams) { + const dataViewId = uuidv4(); + + const columnOrder = [ + DatasetQualityLensColumn.Date, + DatasetQualityLensColumn.CountFailed, + DatasetQualityLensColumn.CountTotal, + DatasetQualityLensColumn.Math, + DatasetQualityLensColumn.FailedDocs, + ]; + + if (breakdownFieldName) { + columnOrder.unshift(DatasetQualityLensColumn.Breakdown); + } + + const columns = getChartColumns(breakdownFieldName); + return { + visualizationType: 'lnsXY', + title: flyoutFailedDocsTrendText, + references: [], + state: { + ...getAdHocDataViewState(dataViewId, dataStream, datasetTitle), + datasourceStates: { + formBased: { + layers: { + layer1: { + columnOrder, + columns, + indexPatternId: dataViewId, + }, + }, + }, + }, + filters: [], + query: { + language: 'kuery', + query: '', + }, + visualization: { + axisTitlesVisibilitySettings: { + x: false, + yLeft: false, + yRight: false, + }, + fittingFunction: 'None', + gridlinesVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + layers: [ + { + accessors: [DatasetQualityLensColumn.FailedDocs], + layerId: 'layer1', + layerType: 'data', + seriesType: 'line', + xAccessor: DatasetQualityLensColumn.Date, + ...(breakdownFieldName + ? { splitAccessor: DatasetQualityLensColumn.Breakdown } + : { + yConfig: [ + { + forAccessor: DatasetQualityLensColumn.FailedDocs, + color, + }, + ], + }), + }, + ], + legend: { + isVisible: true, + position: 'right', + legendSize: 'large', + shouldTruncate: true, + }, + preferredSeriesType: 'line', + tickLabelsVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + valueLabels: 'hide', + yLeftExtent: { + mode: 'custom', + lowerBound: 0, + upperBound: undefined, + }, + }, + }, + } as TypedLensByValueInput['attributes']; +} + +function getAdHocDataViewState(id: string, dataStream: string, title: string) { + // TODO: Need to fix the index pattern used here (aka ::failures) + return { + internalReferences: [ + { + id, + name: 'indexpattern-datasource-current-indexpattern', + type: 'index-pattern', + }, + { + id, + name: 'indexpattern-datasource-layer-layer1', + type: 'index-pattern', + }, + ], + adHocDataViews: { + [id]: { + id, + title: dataStream, + timeFieldName: '@timestamp', + sourceFilters: [], + fieldFormats: {}, + runtimeFieldMap: {}, + fieldAttrs: {}, + allowNoIndex: false, + name: title, + }, + }, + }; +} + +function getChartColumns(breakdownField?: string): Record { + return { + [DatasetQualityLensColumn.Date]: { + dataType: 'date', + isBucketed: true, + label: '@timestamp', + operationType: 'date_histogram', + params: { + interval: 'auto', + }, + scale: 'interval', + sourceField: '@timestamp', + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.CountFailed]: { + label: '', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + filter: { + query: '', + language: 'kuery', + }, + params: { + emptyAsNull: false, + }, + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.CountTotal]: { + label: '', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + params: { + emptyAsNull: false, + }, + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.Math]: { + label: '', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'divide', + args: ['count_failed', 'count_total'], + location: { + min: 0, + max: 34, + }, + text: "count(kql='') / count()", + }, + }, + references: ['count_failed', 'count_total'], + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.FailedDocs]: { + label: flyoutFailedDocsPercentageText, + customLabel: true, + operationType: 'formula', + dataType: 'number', + references: [DatasetQualityLensColumn.Math], + isBucketed: false, + params: { + formula: "count(kql='') / count()", + format: { + id: 'percent', + params: { + decimals: 3, + }, + }, + isFormulaBroken: false, + }, + } as GenericIndexPatternColumn, + ...(breakdownField + ? { + [DatasetQualityLensColumn.Breakdown]: { + dataType: 'number', + isBucketed: true, + label: getFlyoutFailedDocsTopNText(MAX_BREAKDOWN_SERIES, breakdownField), + operationType: 'terms', + scale: 'ordinal', + sourceField: breakdownField, + params: { + size: MAX_BREAKDOWN_SERIES, + orderBy: { + type: 'alphabetical', + fallback: true, + }, + orderDirection: 'desc', + otherBucket: true, + missingBucket: true, + parentFormat: { + id: 'terms', + }, + }, + } as GenericIndexPatternColumn, + } + : {}), + }; +} + +const getFlyoutFailedDocsTopNText = (count: number, fieldName: string) => + i18n.translate('xpack.datasetQuality.details.failedDocsTopNValues', { + defaultMessage: 'Top {count} values of {fieldName}', + values: { count, fieldName }, + description: + 'Tooltip label for the top N values of a field in the failed documents trend chart.', + }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx similarity index 51% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx index e25b1da9540f8..fa88fd1d5589e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx @@ -5,12 +5,12 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, + EuiButtonGroup, EuiButtonIcon, - EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, @@ -22,75 +22,42 @@ import { OnTimeChangeProps, useGeneratedHtmlId, } from '@elastic/eui'; -import type { DataViewField } from '@kbn/data-views-plugin/common'; import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; +import { i18n } from '@kbn/i18n'; import { discoverAriaText, logsExplorerAriaText, openInDiscoverText, openInLogsExplorerText, - overviewDegradedDocsText, -} from '../../../../../../common/translations'; -import { DegradedDocsChart } from './degraded_docs_chart'; -import { - useDatasetDetailsRedirectLinkTelemetry, - useDatasetQualityDetailsState, - useDegradedDocsChart, - useRedirectLink, -} from '../../../../../hooks'; -import { _IGNORED } from '../../../../../../common/es_fields'; -import { NavigationSource } from '../../../../../services/telemetry'; + overviewTrendsDocsText, +} from '../../../../../common/translations'; +import { TrendDocsChart } from './trend_docs_chart'; +import { useDatasetQualityDetailsState, useQualityIssuesDocsChart } from '../../../../hooks'; -const degradedDocsTooltip = ( +const trendDocsTooltip = ( - _ignored - - ), - }} + id="xpack.datasetQuality.details.trendDocsTooltip" + defaultMessage="The percentage of ignored fields or failed docs over the selected timeframe." /> ); // Allow for lazy loading // eslint-disable-next-line import/no-default-export -export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: number }) { - const { timeRange, updateTimeRange, datasetDetails } = useDatasetQualityDetailsState(); - const { dataView, breakdown, ...chartProps } = useDegradedDocsChart(); +export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: number }) { + const { timeRange, updateTimeRange, docsTrendChart } = useDatasetQualityDetailsState(); + const { + dataView, + breakdown, + redirectLinkProps, + handleDocsTrendChartChange, + ...qualityIssuesChartProps + } = useQualityIssuesDocsChart(); const accordionId = useGeneratedHtmlId({ - prefix: overviewDegradedDocsText, - }); - - const [breakdownDataViewField, setBreakdownDataViewField] = useState( - undefined - ); - - const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ - query: { language: 'kuery', query: `${_IGNORED}: *` }, - navigationSource: NavigationSource.Trend, - }); - - const degradedDocLinkLogsExplorer = useRedirectLink({ - dataStreamStat: datasetDetails, - timeRangeConfig: timeRange, - query: { language: 'kuery', query: `${_IGNORED}: *` }, - breakdownField: breakdownDataViewField?.name, - sendTelemetry, + prefix: overviewTrendsDocsText, }); - useEffect(() => { - if (breakdown.dataViewField && breakdown.fieldSupportsBreakdown) { - setBreakdownDataViewField(breakdown.dataViewField); - } else { - setBreakdownDataViewField(undefined); - } - }, [setBreakdownDataViewField, breakdown]); - const onTimeRangeChange = useCallback( ({ start, end }: Pick) => { updateTimeRange({ start, end, refreshInterval: timeRange.refresh.value }); @@ -108,9 +75,9 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe `} > -
{overviewDegradedDocsText}
+
{overviewTrendsDocsText}
- + @@ -125,17 +92,47 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe initialIsOpen={true} data-test-subj="datasetQualityDetailsOverviewDocumentTrends" > + + + handleDocsTrendChartChange(id)} + options={[ + { + id: 'degraded', + label: i18n.translate('xpack.datasetQuality.details.chartType.degradedDocs', { + defaultMessage: 'Ignored fields', + }), + }, + { + id: 'failed', + label: i18n.translate('xpack.datasetQuality.details.chartType.failedDocs', { + defaultMessage: 'Failed docs', + }), + }, + ]} + idSelected={docsTrendChart} + /> + - - - , + ReturnType, 'attributes' | 'isChartLoading' | 'onChartLoading' | 'extraActions' > { timeRange: TimeRangeConfig; @@ -34,7 +34,7 @@ interface DegradedDocsChartProps onTimeRangeChange: (props: Pick) => void; } -export function DegradedDocsChart({ +export function TrendDocsChart({ attributes, isChartLoading, onChartLoading, @@ -42,7 +42,7 @@ export function DegradedDocsChart({ timeRange, lastReloadTime, onTimeRangeChange, -}: DegradedDocsChartProps) { +}: TrendDocsChartProps) { const { services: { lens }, } = useKibanaContextForPlugin(); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx index 380dd6bf09b95..456a43f5b83bf 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx @@ -14,7 +14,7 @@ import { DegradedFields } from './degraded_fields'; const OverviewHeader = dynamic(() => import('./header')); const Summary = dynamic(() => import('./summary')); -const DegradedDocs = dynamic(() => import('./document_trends/degraded_docs')); +const DocumentTrends = dynamic(() => import('./document_trends')); export function Overview() { const { dataStream, isNonAggregatable, updateTimeRange } = useDatasetQualityDetailsState(); @@ -34,7 +34,7 @@ export function Overview() { - + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 897efb821ff64..a1279895b24db 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -6,10 +6,12 @@ */ import React from 'react'; -import { EuiFlexGroup } from '@elastic/eui'; +import { EuiCode, EuiFlexGroup } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { Panel, PanelIndicator } from './panel'; import { overviewPanelDatasetQualityIndicatorDegradedDocs, + overviewPanelDatasetQualityIndicatorFailedDocs, overviewPanelDocumentsIndicatorSize, overviewPanelDocumentsIndicatorTotalCount, overviewPanelResourcesIndicatorServices, @@ -21,6 +23,27 @@ import { import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel'; import { DatasetQualityIndicator } from '../../../quality_indicator'; +const degradedDocsTooltip = ( + + _ignored + + ), + }} + /> +); + +const failedDocsColumnTooltip = ( + +); + // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function Summary() { @@ -32,6 +55,7 @@ export default function Summary() { totalServicesCount, totalHostsCount, totalDegradedDocsCount, + totalFailedDocsCount, quality, } = useOverviewSummaryPanel(); return ( @@ -75,6 +99,13 @@ export default function Summary() { label={overviewPanelDatasetQualityIndicatorDegradedDocs} value={totalDegradedDocsCount} isLoading={isSummaryPanelLoading} + tooltip={degradedDocsTooltip} + /> + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx index e03e0957b5a52..f11bb14be29fd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx @@ -6,7 +6,15 @@ */ import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSkeletonTitle, EuiText } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiPanel, + EuiSkeletonTitle, + EuiText, + EuiToolTip, +} from '@elastic/eui'; import { css } from '@emotion/react'; import { euiThemeVars } from '@kbn/ui-theme'; import { PrivilegesWarningIconWrapper } from '../../../common'; @@ -71,11 +79,13 @@ export function Panel({ export function PanelIndicator({ label, value, + tooltip, isLoading, userHasPrivilege = true, }: { label: string; value: string | number; + tooltip?: React.ReactElement; isLoading: boolean; userHasPrivilege?: boolean; }) { @@ -85,9 +95,23 @@ export function PanelIndicator({ ) : ( <> - - {label} - + {tooltip ? ( + + + {`${label} `} + + + + ) : ( + + {label} + + )} { +export const DegradedFieldInfo = () => { const { - fieldFormats, degradedFieldValues, - isDegradedFieldsLoading, isAnalysisInProgress, degradedFieldAnalysisFormattedResult, degradedFieldAnalysis, } = useDegradedFields(); - const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ - ES_FIELD_TYPES.DATE, - ]); + console.log({ + degradedFieldValues, + isAnalysisInProgress, + degradedFieldAnalysisFormattedResult, + degradedFieldAnalysis, + }); return ( - - - - - {countColumnName} - - - - - - - - - - - {lastOccurrenceColumnName} - - - - {dateFormatter.convert(fieldList?.lastOccurrence)} - - - - + <> @@ -175,6 +133,6 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) )} - + ); }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/index.tsx new file mode 100644 index 0000000000000..2332f9f17e0c4 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/index.tsx @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useMemo } from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../hooks'; +import { QualityIssueFieldInfo } from '../field_info'; +import { PossibleDegradedFieldMitigations } from './possible_mitigations'; +import { DegradedFieldInfo } from './field_info'; + +// Allow for lazy loading +// eslint-disable-next-line import/no-default-export +export default function DegradedFieldFlyout() { + const { expandedDegradedField, renderedItems } = useDegradedFields(); + const { dataStreamSettings } = useDatasetQualityDetailsState(); + + const fieldList = useMemo(() => { + return renderedItems.find((item) => { + return item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type; + }); + }, [renderedItems, expandedDegradedField]); + + const isUserViewingTheIssueOnLatestBackingIndex = + dataStreamSettings?.lastBackingIndexName === fieldList?.indexFieldWasLastPresentIn; + + return ( + <> + + + + {isUserViewingTheIssueOnLatestBackingIndex && ( + <> + + + + )} + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx similarity index 88% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx index 0dd80bb120e54..954b65ce6c8bf 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/field_limit_documentation_link.tsx @@ -7,8 +7,8 @@ import React from 'react'; import { EuiLink } from '@elastic/eui'; -import { useKibanaContextForPlugin } from '../../../../../utils'; -import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../common/translations'; +import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../../common/translations'; +import { useKibanaContextForPlugin } from '../../../../../../utils'; export function FieldLimitDocLink() { const { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx similarity index 95% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx index 1056713ac2070..864bc26884354 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -15,6 +15,7 @@ import { EuiTitle, useGeneratedHtmlId, } from '@elastic/eui'; +import { useDegradedFields } from '../../../../../../hooks'; import { fieldLimitMitigationConsiderationText, fieldLimitMitigationConsiderationText1, @@ -23,8 +24,7 @@ import { fieldLimitMitigationConsiderationText4, fieldLimitMitigationDescriptionText, increaseFieldMappingLimitTitle, -} from '../../../../../../common/translations'; -import { useDegradedFields } from '../../../../../hooks'; +} from '../../../../../../../common/translations'; import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit'; import { FieldLimitDocLink } from './field_limit_documentation_link'; import { MessageCallout } from './message_callout'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx similarity index 96% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx index 158a5e5eba460..a9ebbedfb6c15 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx @@ -14,13 +14,13 @@ import { EuiButton, EuiFieldNumber, } from '@elastic/eui'; +import { useDegradedFields } from '../../../../../../hooks'; import { fieldLimitMitigationApplyButtonText, fieldLimitMitigationCurrentLimitLabelText, fieldLimitMitigationNewLimitButtonText, fieldLimitMitigationNewLimitPlaceholderText, -} from '../../../../../../common/translations'; -import { useDegradedFields } from '../../../../../hooks'; +} from '../../../../../../../common/translations'; export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit: number }) { // Propose the user a 30% increase over the current limit diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx similarity index 94% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx index 168ae4df575e9..66f87b2d92082 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/field_limit/message_callout.tsx @@ -15,10 +15,10 @@ import { fieldLimitMitigationRolloverButton, fieldLimitMitigationSuccessComponentTemplateLinkText, fieldLimitMitigationSuccessMessage, -} from '../../../../../../common/translations'; -import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; -import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; -import { useKibanaContextForPlugin } from '../../../../../utils'; +} from '../../../../../../../common/translations'; +import { useKibanaContextForPlugin } from '../../../../../../utils'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../../common/utils/component_template_name'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../../hooks'; export function MessageCallout() { const { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/index.tsx similarity index 50% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/index.tsx index 34f39f25a67ec..676358e250085 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/degraded_field_flyout/possible_mitigations/index.tsx @@ -7,29 +7,23 @@ import React from 'react'; import { EuiSpacer } from '@elastic/eui'; -import { ManualMitigations } from './manual'; import { FieldMappingLimit } from './field_limit/field_mapping_limit'; -import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../hooks'; -import { PossibleMitigationTitle } from './title'; +import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks'; +import { PossibleMitigations } from '../../possible_mitigations'; -export function PossibleMitigations() { - const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields(); +export function PossibleDegradedFieldMitigations() { + const { degradedFieldAnalysis } = useDegradedFields(); const { integrationDetails } = useDatasetQualityDetailsState(); const isIntegration = Boolean(integrationDetails?.integration); return ( - !isAnalysisInProgress && ( -
- - - {degradedFieldAnalysis?.isFieldLimitIssue && ( - <> - - - - )} - -
- ) + + {degradedFieldAnalysis?.isFieldLimitIssue && ( + <> + + + + )} + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/columns.tsx new file mode 100644 index 0000000000000..d73298ca5a75e --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/columns.tsx @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiBadge, + EuiBasicTableColumn, + EuiCode, + EuiCodeBlock, + EuiIcon, + EuiToolTip, +} from '@elastic/eui'; +import { FailedDocsErrors } from '../../../../../common/api_types'; + +const contentColumnName = i18n.translate( + 'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.contentLabel', + { + defaultMessage: 'Content', + } +); + +const typeColumnName = i18n.translate( + 'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.typeLabel', + { + defaultMessage: 'Type', + } +); + +const typeColumnTooltip = i18n.translate( + 'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.typeTooltip', + { + defaultMessage: 'Error message category', + } +); + +type FailedDocsError = FailedDocsErrors['errors']; + +export const getFailedDocsErrorsColumns = (): Array> => [ + { + name: contentColumnName, + field: 'message', + render: (_, { message }) => { + return {message}; + }, + }, + { + name: ( + + + {`${typeColumnName} `} + + + + ), + field: 'type', + render: (_, { type }) => { + return ( + + {type} + + ); + }, + }, +]; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/field_info.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/field_info.tsx new file mode 100644 index 0000000000000..56b2c142a6e68 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/field_info.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiBasicTable, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiSpacer, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { + failedDocsErrorsColumnName, + overviewDegradedFieldsTableLoadingText, +} from '../../../../../common/translations'; +import { useDegradedFields } from '../../../../hooks'; + +const failedDocsErrorsTableNoData = i18n.translate( + 'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.noData', + { + defaultMessage: 'No errors found', + } +); + +export const FailedFieldInfo = () => { + const { + isDegradedFieldsLoading, + failedDocsErrorsColumns, + renderedFailedDocsErrorsItems, + failedDocsErrorsSort, + isFailedDocsErrorsLoading, + resultsCount, + } = useDegradedFields(); + + return ( + <> + + + + {failedDocsErrorsColumnName} + + + + + + + + + { + console.log(e); + }} + data-test-subj="datasetQualityDetailsDegradedFieldTable" + rowProps={{ + 'data-test-subj': 'datasetQualityDetailsDegradedTableRow', + }} + noItemsMessage={ + isDegradedFieldsLoading + ? overviewDegradedFieldsTableLoadingText + : failedDocsErrorsTableNoData + } + pagination={{ + pageIndex: 0, + pageSize: 5, + totalItemCount: 5, + }} + /> + + + + + ); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/index.tsx new file mode 100644 index 0000000000000..2e708243d3a61 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useMemo } from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { useDegradedFields } from '../../../../hooks'; +import { QualityIssueFieldInfo } from '../field_info'; +import { PossibleMitigations } from '../possible_mitigations'; +import { FailedFieldInfo } from './field_info'; + +// Allow for lazy loading +// eslint-disable-next-line import/no-default-export +export default function FailedDocsFlyout() { + const { expandedDegradedField, renderedItems } = useDegradedFields(); + + const fieldList = useMemo(() => { + return renderedItems.find((item) => { + return item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type; + }); + }, [renderedItems, expandedDegradedField]); + + return ( + <> + + + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/field_info.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/field_info.tsx new file mode 100644 index 0000000000000..35d02b0ee442b --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/field_info.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule, EuiTitle, formatNumber } from '@elastic/eui'; +import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; + +import { NUMBER_FORMAT } from '../../../../common/constants'; +import { countColumnName, lastOccurrenceColumnName } from '../../../../common/translations'; +import { useDegradedFields } from '../../../hooks'; +import { SparkPlot } from '../../common/spark_plot'; +import { QualityIssue } from '../../../../common/api_types'; + +export const QualityIssueFieldInfo = ({ + fieldList, + children, +}: { + fieldList?: QualityIssue; + children?: React.ReactNode; +}) => { + const { fieldFormats, isDegradedFieldsLoading } = useDegradedFields(); + + const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ + ES_FIELD_TYPES.DATE, + ]); + + return ( + + + + + {countColumnName} + + + + + + + + + + + {lastOccurrenceColumnName} + + + + {dateFormatter.convert(fieldList?.lastOccurrence)} + + + + {children} + + ); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/index.tsx similarity index 78% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/index.tsx index bb72b4f6de20f..38b03a9d07748 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/index.tsx @@ -38,13 +38,13 @@ import { openInLogsExplorerText, overviewDegradedFieldsSectionTitle, } from '../../../../common/translations'; -import { DegradedFieldInfo } from './field_info'; import { _IGNORED } from '../../../../common/es_fields'; -import { PossibleMitigations } from './possible_mitigations'; +import DegradedFieldFlyout from './degraded_field_flyout'; +import FailedDocsFlyout from './failed_docs_flyout'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export -export default function DegradedFieldFlyout() { +export default function QualityIssueFlyout() { const { closeDegradedFieldFlyout, expandedDegradedField, @@ -59,7 +59,7 @@ export default function DegradedFieldFlyout() { const fieldList = useMemo(() => { return renderedItems.find((item) => { - return item.name === expandedDegradedField; + return item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type; }); }, [renderedItems, expandedDegradedField]); @@ -92,7 +92,14 @@ export default function DegradedFieldFlyout() { - {expandedDegradedField} {fieldIgnoredText} + {expandedDegradedField?.type === 'degraded' ? ( + <> + {expandedDegradedField?.name}{' '} + {fieldIgnoredText} + + ) : ( + {'Documents indexing failed'} + )} - {!isUserViewingTheIssueOnLatestBackingIndex && ( - <> - - - {degradedFieldMessageIssueDoesNotExistInLatestIndex} - - - )} - {isUserViewingTheIssueOnLatestBackingIndex && + {expandedDegradedField?.type === 'degraded' && + !isUserViewingTheIssueOnLatestBackingIndex && ( + <> + + + {degradedFieldMessageIssueDoesNotExistInLatestIndex} + + + )} + {expandedDegradedField?.type === 'degraded' && + isUserViewingTheIssueOnLatestBackingIndex && !isAnalysisInProgress && degradedFieldAnalysisFormattedResult && !degradedFieldAnalysisFormattedResult.identifiedUsingHeuristics && ( @@ -156,13 +165,8 @@ export default function DegradedFieldFlyout() { )} - - {isUserViewingTheIssueOnLatestBackingIndex && ( - <> - - - - )} + {expandedDegradedField?.type === 'degraded' && } + {expandedDegradedField?.type === 'failed' && } ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/index.tsx new file mode 100644 index 0000000000000..46e6483c80cb9 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/index.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { ManualMitigations } from './manual'; +import { useDegradedFields } from '../../../../hooks'; +import { PossibleMitigationTitle } from './title'; + +export function PossibleMitigations({ children }: { children?: React.ReactNode }) { + const { isAnalysisInProgress } = useDegradedFields(); + + return ( + !isAnalysisInProgress && ( +
+ + + + {children} + +
+ ) + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/manual/component_template_link.tsx similarity index 100% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/manual/component_template_link.tsx diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/manual/index.tsx similarity index 100% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/manual/index.tsx diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/manual/pipeline_link.tsx similarity index 100% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/manual/pipeline_link.tsx diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/title.tsx similarity index 100% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/title.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/quality_issue_flyout/possible_mitigations/title.tsx diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts deleted file mode 100644 index c9588da392d96..0000000000000 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { countBy } from 'lodash'; -import { QualityIndicators } from '../../../common/types'; -import { mapPercentageToQuality } from '../../../common/utils'; - -export const mapPercentagesToQualityCounts = ( - percentages: number[] -): Record => - countBy(percentages.map(mapPercentageToQuality)) as Record; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts index 4f41732ca5259..f95d565f1b217 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts @@ -7,5 +7,4 @@ export * from './indicator'; export * from './percentage_indicator'; -export * from './helpers'; export * from './dataset_quality_indicator'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx index c10a96672598a..8f607b65b5e9d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx @@ -10,43 +10,46 @@ import { i18n } from '@kbn/i18n'; import { FormattedNumber } from '@kbn/i18n-react'; import React from 'react'; -const FEW_DEGRADED_DOCS_THRESHOLD = 0.0005; +const FEW_DOCS_THRESHOLD = 0.0005; export function QualityPercentageIndicator({ percentage, - degradedDocsCount, + docsCount, + tooltipContent, }: { percentage: number; - degradedDocsCount?: number; + docsCount?: number; + tooltipContent: (numberOfDocuments: number) => string; }) { - const isFewDegradedDocsAvailable = percentage && percentage < FEW_DEGRADED_DOCS_THRESHOLD; + const isFewDocsAvailable = percentage && percentage < FEW_DOCS_THRESHOLD; - return isFewDegradedDocsAvailable ? ( - + return isFewDocsAvailable ? ( + ) : ( - + ); } -const DatasetWithFewDegradedDocs = ({ degradedDocsCount }: { degradedDocsCount?: number }) => { +const DatasetWithFewDocs = ({ + docsCount, + tooltipContent, +}: { + docsCount: number; + tooltipContent: (numberOfDocuments: number) => string; +}) => { return ( - ~0%{' '} - + {i18n.translate('xpack.datasetQuality.datasetWithFewDocs.TextLabel', { + defaultMessage: '~0%', + })}{' '} + ); }; -const DatasetWithManyDegradedDocs = ({ percentage }: { percentage: number }) => { +const DatasetWithManyDocs = ({ percentage }: { percentage: number }) => { return ( % diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts index a87712a5e364e..c52f866ecc8f8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts @@ -20,8 +20,9 @@ export const getPublicStateFromContext = ( degradedFields: context.degradedFields, timeRange: context.timeRange, breakdownField: context.breakdownField, + qualityIssuesChart: context.qualityIssuesChart, integration: context.integration, - expandedDegradedField: context.expandedDegradedField, + expandedQualityIssue: context.expandedQualityIssue, showCurrentQualityIssues: context.showCurrentQualityIssues, }; }; @@ -51,8 +52,9 @@ export const getContextFromPublicState = ( }, }, dataStream: publicState.dataStream, + qualityIssuesChart: publicState.qualityIssuesChart ?? DEFAULT_CONTEXT.qualityIssuesChart, breakdownField: publicState.breakdownField, - expandedDegradedField: publicState.expandedDegradedField, + expandedQualityIssue: publicState.expandedQualityIssue, showCurrentQualityIssues: publicState.showCurrentQualityIssues ?? DEFAULT_CONTEXT.showCurrentQualityIssues, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts index 1f9397ee4504c..eb99e5cf4dec9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts @@ -30,7 +30,11 @@ export type DatasetQualityDetailsPublicState = WithDefaultControllerState; export type DatasetQualityDetailsPublicStateUpdate = Partial< Pick< WithDefaultControllerState, - 'timeRange' | 'breakdownField' | 'expandedDegradedField' | 'showCurrentQualityIssues' + | 'timeRange' + | 'breakdownField' + | 'expandedQualityIssue' + | 'showCurrentQualityIssues' + | 'qualityIssuesChart' > > & { dataStream: string; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts index ad588fd0b673f..59c3480a24518 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts @@ -6,7 +6,7 @@ */ export * from './use_dataset_quality_table'; -export * from './use_degraded_docs_chart'; +export * from './use_quality_issues_docs_chart'; export * from './use_redirect_link'; export * from './use_summary_panel'; export * from './use_create_dataview'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts index f613d3af7fdc4..77ccc5fd008cd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts @@ -15,6 +15,7 @@ import { DataStreamDetails } from '../../common/api_types'; import { Integration } from '../../common/data_streams_stats/integration'; import { mapPercentageToQuality } from '../../common/utils'; import { MASKED_FIELD_PLACEHOLDER, UNKOWN_FIELD_PLACEHOLDER } from '../../common/constants'; +import { calculatePercentage } from '../utils'; export function useDatasetDetailsTelemetry() { const { @@ -167,10 +168,11 @@ function getDatasetDetailsEbtProps({ type: datasetDetails.type, }; const degradedDocs = dataStreamDetails?.degradedDocsCount ?? 0; - const totalDocs = dataStreamDetails?.docsCount ?? 0; - const degradedPercentage = - totalDocs > 0 ? Number(((degradedDocs / totalDocs) * 100).toFixed(2)) : 0; - const health = mapPercentageToQuality(degradedPercentage); + const failedDocs = dataStreamDetails?.failedDocsCount ?? 0; + const totalDocs = (dataStreamDetails?.docsCount ?? 0) + failedDocs; + const degradedPercentage = calculatePercentage({ totalDocs, count: degradedDocs }); + const failedPercentage = calculatePercentage({ totalDocs, count: failedDocs }); + const health = mapPercentageToQuality([degradedPercentage, failedPercentage]); const { startDate: from, endDate: to } = getDateISORange(timeRange); return { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts index edd16652374a1..2c19c29692b1b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts @@ -27,7 +27,7 @@ export const useDatasetQualityDetailsState = () => { timeRange, breakdownField, isIndexNotFoundError, - expandedDegradedField, + expandedQualityIssue: expandedDegradedField, } = useSelector(service, (state) => state.context) ?? {}; const isNonAggregatable = useSelector(service, (state) => @@ -100,6 +100,8 @@ export const useDatasetQualityDetailsState = () => { rawName: dataStream, }; + const docsTrendChart = useSelector(service, (state) => state.context.qualityIssuesChart); + const loadingState = useSelector(service, (state) => ({ nonAggregatableDatasetLoading: state.matches('initializing.nonAggregatableDataset.fetching'), dataStreamDetailsLoading: state.matches('initializing.dataStreamDetails.fetching'), @@ -144,6 +146,7 @@ export const useDatasetQualityDetailsState = () => { datasetDetails, degradedFields, dataStreamDetails, + docsTrendChart, breakdownField, isBreakdownFieldEcs, isBreakdownFieldAsserted, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 6529ae1841ee3..e61706bdf9813 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -25,6 +25,7 @@ const sortingOverrides: Partial<{ }> = { ['title']: 'name', ['size']: DataStreamStat.calculateFilteredSize, + ['quality']: (item) => Math.max(item.degradedDocs.percentage, item.failedDocs.percentage), }; export const useDatasetQualityTable = () => { @@ -65,15 +66,23 @@ export const useDatasetQualityTable = () => { service, (state) => state.matches('stats.datasets.fetching') || + state.matches('stats.docsStats.fetching') || state.matches('integrations.fetching') || - state.matches('stats.degradedDocs.fetching') + state.matches('stats.degradedDocs.fetching') || + state.matches('stats.failedDocs.fetching') ); const loadingDataStreamStats = useSelector(service, (state) => state.matches('stats.datasets.fetching') ); + const loadingDocStats = useSelector(service, (state) => + state.matches('stats.docsStats.fetching') + ); const loadingDegradedStats = useSelector(service, (state) => state.matches('stats.degradedDocs.fetching') ); + const loadingFailedStats = useSelector(service, (state) => + state.matches('stats.failedDocs.fetching') + ); const datasets = useSelector(service, (state) => state.context.datasets); @@ -99,7 +108,9 @@ export const useDatasetQualityTable = () => { canUserMonitorDataset, canUserMonitorAnyDataStream, loadingDataStreamStats, + loadingDocStats, loadingDegradedStats, + loadingFailedStats, showFullDatasetNames, isActiveDataset: isActive, timeRange, @@ -110,7 +121,9 @@ export const useDatasetQualityTable = () => { canUserMonitorDataset, canUserMonitorAnyDataStream, loadingDataStreamStats, + loadingDocStats, loadingDegradedStats, + loadingFailedStats, showFullDatasetNames, isActive, timeRange, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.tsx similarity index 54% rename from x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts rename to x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.tsx index 49ceb50abc3cd..866608282b4c2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.tsx @@ -4,8 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import React, { useCallback, useMemo } from 'react'; import { useSelector } from '@xstate/react'; -import { useCallback, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; import { orderBy } from 'lodash'; import { DegradedField } from '../../common/data_streams_stats'; import { SortDirection } from '../../common/types'; @@ -23,6 +24,8 @@ import { degradedFieldCauseFieldMalformed, degradedFieldCauseFieldMalformedTooltip, } from '../../common/translations'; +import { QualityIssueType } from '../state_machines/dataset_quality_details_controller'; +import { getFailedDocsErrorsColumns } from '../components/dataset_quality_details/quality_issue_flyout/failed_docs_flyout/columns'; export type DegradedFieldSortField = keyof DegradedField; @@ -32,13 +35,22 @@ export function useDegradedFields() { services: { fieldFormats }, } = useKibanaContextForPlugin(); - const { degradedFields, expandedDegradedField, showCurrentQualityIssues } = useSelector( - service, - (state) => state.context - ); + const { + degradedFields, + expandedQualityIssue: expandedDegradedField, + showCurrentQualityIssues, + failedDocsErrors, + } = useSelector(service, (state) => state.context); const { data, table } = degradedFields ?? {}; const { page, rowsPerPage, sort } = table; + const { data: failedDocsErrorsData, table: failedDocsErrorsTable } = failedDocsErrors ?? {}; + const { + page: failedDocsErrorsPage, + rowsPerPage: failedDocsErrorsRowsPerPage, + sort: failedDocsErrorsSort, + } = failedDocsErrorsTable; + const totalItemCount = data?.length ?? 0; const pagination = { @@ -74,7 +86,10 @@ export function useDegradedFields() { }, [data, sort.field, sort.direction, page, rowsPerPage]); const expandedRenderedItem = useMemo(() => { - return renderedItems.find((item) => item.name === expandedDegradedField); + return renderedItems.find( + (item) => + item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type + ); }, [expandedDegradedField, renderedItems]); const isDegradedFieldsLoading = useSelector(service, (state) => @@ -89,11 +104,20 @@ export function useDegradedFields() { ); const openDegradedFieldFlyout = useCallback( - (fieldName: string) => { - if (expandedDegradedField === fieldName) { + (fieldName: string, qualityIssueType: QualityIssueType) => { + if ( + expandedDegradedField?.name === fieldName && + expandedDegradedField?.type === qualityIssueType + ) { service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' }); } else { - service.send({ type: 'OPEN_DEGRADED_FIELD_FLYOUT', fieldName }); + service.send({ + type: 'OPEN_DEGRADED_FIELD_FLYOUT', + qualityIssue: { + name: fieldName, + type: qualityIssueType, + }, + }); } }, [expandedDegradedField, service] @@ -103,23 +127,39 @@ export function useDegradedFields() { service.send('TOGGLE_CURRENT_QUALITY_ISSUES'); }, [service]); - const degradedFieldValues = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.initialized.ignoredValues.done') - ? state.context.degradedFieldValues - : undefined + const degradedFieldValues = useSelector( + service, + (state) => + /* state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.ignoredValues.done' + ) + ? */ state.context.degradedFieldValues + /* : undefined */ ); - const degradedFieldAnalysis = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed') || - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating') || + const degradedFieldAnalysis = useSelector( + service, + (state) => + /* state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.analyzed' + ) || state.matches( - 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.mitigating' ) || - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver') || - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') || - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error') - ? state.context.degradedFieldAnalysis - : undefined + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.askingForRollover' + ) || + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.rollingOver' + ) || + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.success' + ) || + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.error' + ) + ? */ state.context.degradedFieldAnalysis + /* : undefined */ ); const degradedFieldAnalysisFormattedResult = useMemo(() => { @@ -165,27 +205,37 @@ export function useDegradedFields() { const isDegradedFieldsValueLoading = useSelector(service, (state) => { return state.matches( - 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.ignoredValues.fetching' + ); + }); + + const isFailedDocsErrorsLoading = useSelector(service, (state) => { + return state.matches( + 'initializing.degradedFieldFlyout.open.failedDocsFlyout.initialized.failedDocsErrors.fetching' ); }); const isRolloverRequired = useSelector(service, (state) => { return state.matches( - 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.askingForRollover' ); }); const isMitigationAppliedSuccessfully = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success'); + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.success' + ); }); const isAnalysisInProgress = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing'); + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.analyzing' + ); }); const isRolloverInProgress = useSelector(service, (state) => { return state.matches( - 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver' + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.rollingOver' ); }); @@ -197,12 +247,18 @@ export function useDegradedFields() { ); const isMitigationInProgress = useSelector(service, (state) => { - return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating'); + return state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.mitigating' + ); }); const newFieldLimitData = useSelector(service, (state) => - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') || - state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error') + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.success' + ) || + state.matches( + 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.error' + ) ? state.context.fieldLimit : undefined ); @@ -211,6 +267,54 @@ export function useDegradedFields() { service.send('ROLLOVER_DATA_STREAM'); }, [service]); + const failedDocsErrorsColumns = useMemo(() => getFailedDocsErrorsColumns(), []); + + const renderedFailedDocsErrorsItems = useMemo(() => { + const sortedItems = orderBy( + failedDocsErrorsData, + failedDocsErrorsSort.field, + failedDocsErrorsSort.direction + ); + return sortedItems.slice( + failedDocsErrorsPage * failedDocsErrorsRowsPerPage, + (failedDocsErrorsPage + 1) * failedDocsErrorsRowsPerPage + ); + }, [ + failedDocsErrorsData, + failedDocsErrorsSort.field, + failedDocsErrorsSort.direction, + failedDocsErrorsPage, + failedDocsErrorsRowsPerPage, + ]); + + const resultsCount = useMemo(() => { + const startNumberItemsOnPage = + failedDocsErrorsRowsPerPage * failedDocsErrorsPage + + (renderedFailedDocsErrorsItems.length ? 1 : 0); + const endNumberItemsOnPage = + failedDocsErrorsRowsPerPage * failedDocsErrorsPage + renderedFailedDocsErrorsItems.length; + + return failedDocsErrorsRowsPerPage === 0 ? ( + + {i18n.translate('xpack.datasetQuality.resultsCount.strong.lllLabel', { + defaultMessage: 'lll', + })} + + ) : ( + <> + + {startNumberItemsOnPage}-{endNumberItemsOnPage} + {' '} + {' of '} {failedDocsErrorsData?.length} + + ); + }, [ + failedDocsErrorsRowsPerPage, + failedDocsErrorsPage, + renderedFailedDocsErrorsItems.length, + failedDocsErrorsData?.length, + ]); + return { isDegradedFieldsLoading, pagination, @@ -237,5 +341,10 @@ export function useDegradedFields() { isRolloverRequired, isMitigationAppliedSuccessfully, triggerRollover, + isFailedDocsErrorsLoading, + failedDocsErrorsColumns, + renderedFailedDocsErrorsItems, + failedDocsErrorsSort: { sort: failedDocsErrorsSort }, + resultsCount, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts index 43cf6923075ee..98e7aba30d493 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts @@ -55,12 +55,19 @@ export const useOverviewSummaryPanel = () => { NUMBER_FORMAT ); + const totalFailedDocsCount = formatNumber(dataStreamDetails?.failedDocsCount ?? 0, NUMBER_FORMAT); + const degradedPercentage = - Number(totalDocsCount) > 0 - ? (Number(totalDegradedDocsCount) / Number(totalDocsCount)) * 100 + (dataStreamDetails.docsCount ?? 0) > 0 + ? ((dataStreamDetails?.degradedDocsCount ?? 0) / dataStreamDetails.docsCount!) * 100 + : 0; + + const failedPercentage = + (dataStreamDetails.docsCount ?? 0) > 0 + ? ((dataStreamDetails?.failedDocsCount ?? 0) / dataStreamDetails.docsCount!) * 100 : 0; - const quality = mapPercentageToQuality(degradedPercentage); + const quality = mapPercentageToQuality([degradedPercentage, failedPercentage]); return { totalDocsCount, @@ -70,6 +77,7 @@ export const useOverviewSummaryPanel = () => { totalHostsCount, isSummaryPanelLoading, totalDegradedDocsCount, + totalFailedDocsCount, quality, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts similarity index 67% rename from x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts rename to x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts index fb57e6e87a74f..d425198bb46c6 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts @@ -15,16 +15,21 @@ import { DEFAULT_LOGS_DATA_VIEW } from '../../common/constants'; import { useCreateDataView } from './use_create_dataview'; import { useKibanaContextForPlugin } from '../utils'; import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state'; -import { getLensAttributes } from '../components/dataset_quality_details/overview/document_trends/degraded_docs/lens_attributes'; +import { getLensAttributes as getDegradedLensAttributes } from '../components/dataset_quality_details/overview/document_trends/degraded_docs/lens_attributes'; +import { getLensAttributes as getFailedLensAttributes } from '../components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes'; +import { useRedirectLink } from './use_redirect_link'; import { useDatasetDetailsTelemetry } from './use_dataset_details_telemetry'; +import { useDatasetDetailsRedirectLinkTelemetry } from './use_redirect_link_telemetry'; +import { QualityIssueType } from '../state_machines/dataset_quality_details_controller'; const openInLensText = i18n.translate('xpack.datasetQuality.details.chartOpenInLensText', { defaultMessage: 'Open in Lens', }); const ACTION_OPEN_IN_LENS = 'ACTION_OPEN_IN_LENS'; +const DEGRADED_DOCS_KUERY = `_ignored:*`; -export const useDegradedDocsChart = () => { +export const useQualityIssuesDocsChart = () => { const { euiTheme } = useEuiTheme(); const { services: { lens }, @@ -34,6 +39,7 @@ export const useDegradedDocsChart = () => { dataStream, datasetDetails, timeRange, + docsTrendChart, breakdownField, integrationDetails, isBreakdownFieldAsserted, @@ -47,9 +53,11 @@ export const useDegradedDocsChart = () => { } = useDatasetDetailsTelemetry(); const [isChartLoading, setIsChartLoading] = useState(undefined); - const [attributes, setAttributes] = useState | undefined>( - undefined - ); + const [attributes, setAttributes] = useState< + ReturnType | undefined + >(undefined); + + const query = docsTrendChart === 'degraded' ? DEGRADED_DOCS_KUERY : ''; const { dataView } = useCreateDataView({ indexPatternString: getDataViewIndexPattern(dataStream), @@ -74,27 +82,47 @@ export const useDegradedDocsChart = () => { [service] ); + const handleDocsTrendChartChange = useCallback( + (qualityIssuesChart: string) => { + service.send({ + type: 'QUALITY_ISSUES_CHART_CHANGE', + qualityIssuesChart, + }); + }, + [service] + ); + useEffect(() => { if (isBreakdownFieldAsserted) trackDatasetDetailsBreakdownFieldChanged(); }, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]); useEffect(() => { + // TODO: Fix dataStreamName for accesing failure store (::failures) const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW; const datasetTitle = integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; - const lensAttributes = getLensAttributes({ - color: euiTheme.colors.danger, - dataStream: dataStreamName, - datasetTitle, - breakdownFieldName: breakdownDataViewField?.name, - }); + const lensAttributes = + docsTrendChart === 'degraded' + ? getDegradedLensAttributes({ + color: euiTheme.colors.danger, + dataStream: dataStreamName, + datasetTitle, + breakdownFieldName: breakdownDataViewField?.name, + }) + : getFailedLensAttributes({ + color: euiTheme.colors.danger, + dataStream: dataStreamName, + datasetTitle, + breakdownFieldName: breakdownDataViewField?.name, + }); setAttributes(lensAttributes); }, [ breakdownDataViewField?.name, euiTheme.colors.danger, setAttributes, dataStream, + docsTrendChart, integrationDetails?.integration?.datasets, datasetDetails.name, ]); @@ -137,6 +165,19 @@ export const useDegradedDocsChart = () => { }; }, [openInLensCallback]); + const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ + query: { language: 'kuery', query }, + navigationSource: navigationSources.Chart, + }); + + const redirectLinkProps = useRedirectLink({ + dataStreamStat: datasetDetails, + query: { language: 'kuery', query }, + timeRangeConfig: timeRange, + breakdownField: breakdownDataViewField?.name, + sendTelemetry, + }); + const extraActions: Action[] = [getOpenInLensAction]; const breakdown = useMemo(() => { @@ -155,12 +196,15 @@ export const useDegradedDocsChart = () => { breakdown, extraActions, isChartLoading, + redirectLinkProps, + handleDocsTrendChartChange, onChartLoading: handleChartLoading, setAttributes, setIsChartLoading, }; }; +// TODO: Fix dataView for accesing failure store (::failures) function getDataViewIndexPattern(dataStream: string | undefined) { return dataStream ?? DEFAULT_LOGS_DATA_VIEW; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts index 8b8b92404d8d5..71eb175575b63 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts @@ -7,10 +7,12 @@ import createContainer from 'constate'; import { useSelector } from '@xstate/react'; +import { countBy } from 'lodash'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { useDatasetQualityTable } from '.'; import { useDatasetQualityContext } from '../components/dataset_quality/context'; import { filterInactiveDatasets } from '../utils'; +import { QualityIndicators } from '../../common/types'; const useSummaryPanel = () => { const { service } = useDatasetQualityContext(); @@ -27,9 +29,10 @@ const useSummaryPanel = () => { Datasets Quality */ - const datasetsQuality = { - percentages: filteredItems.map((item) => item.degradedDocs.percentage), - }; + const datasetsQuality = countBy(filteredItems.map((item) => item.quality)) as Record< + QualityIndicators, + number + >; const isDegradedDocsLoading = useSelector(service, (state) => state.matches('stats.degradedDocs.fetching') diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 827cd4b0a1e49..9883476521a3a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -14,6 +14,8 @@ import { degradedFieldAnalysisRt, DegradedFieldValues, degradedFieldValuesRt, + FailedDocsDetails, + failedDocsErrorsRt, getDataStreamDegradedFieldsResponseRt, getDataStreamsDetailsResponseRt, getDataStreamsSettingsResponseRt, @@ -21,6 +23,7 @@ import { IntegrationDashboardsResponse, integrationDashboardsRT, IntegrationResponse, + qualityIssueBaseRT, UpdateFieldLimitResponse, updateFieldLimitResponseRt, } from '../../../common/api_types'; @@ -28,10 +31,11 @@ import { DataStreamDetails, DataStreamSettings, DegradedFieldResponse, - GetDataStreamDegradedFieldsParams, GetDataStreamDegradedFieldValuesPathParams, GetDataStreamDetailsParams, GetDataStreamDetailsResponse, + GetDataStreamFailedDocsDetailsParams, + GetDataStreamFailedDocsErrorsParams, GetDataStreamSettingsParams, GetDataStreamSettingsResponse, GetIntegrationDashboardsParams, @@ -87,11 +91,64 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { return dataStreamDetails as DataStreamDetails; } + public async getFailedDocsDetails({ + dataStream, + start, + end, + }: GetDataStreamFailedDocsDetailsParams) { + const response = await this.http + .get(`/internal/dataset_quality/data_streams/${dataStream}/failed_docs`, { + query: { start, end }, + }) + .catch((error) => { + throw new DatasetQualityError( + `Failed to fetch data stream failed docs details": ${error}`, + error + ); + }); + + return decodeOrThrow( + qualityIssueBaseRT, + (message: string) => + new DatasetQualityError( + `Failed to decode data stream failed docs details response: ${message}"` + ) + )(response); + } + + public async getFailedDocsErrors({ + dataStream, + start, + end, + }: GetDataStreamFailedDocsErrorsParams): Promise<{ errors: Record }> { + const response = await this.http + .get( + `/internal/dataset_quality/data_streams/${dataStream}/failed_docs/errors`, + { + query: { start, end }, + } + ) + .catch((error) => { + throw new DatasetQualityError( + `Failed to fetch data stream failed docs details": ${error}`, + error + ); + }); + + return decodeOrThrow( + failedDocsErrorsRt, + (message: string) => + new DatasetQualityError( + `Failed to decode data stream failed docs details response: ${message}"` + ) + )(response); + } + public async getDataStreamDegradedFields({ dataStream, start, end, - }: GetDataStreamDegradedFieldsParams) { + }: GetDataStreamFailedDocsDetailsParams) { const response = await this.http .get( `/internal/dataset_quality/data_streams/${dataStream}/degraded_fields`, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index 6eac8bd732840..8010291d8438b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -16,6 +16,8 @@ import { GetDataStreamDegradedFieldsParams, DegradedFieldResponse, GetDataStreamDegradedFieldValuesPathParams, + GetDataStreamFailedDocsDetailsParams, + GetDataStreamFailedDocsErrorsParams, } from '../../../common/data_streams_stats'; import { AnalyzeDegradedFieldsParams, @@ -27,6 +29,7 @@ import { DataStreamRolloverResponse, DegradedFieldAnalysis, DegradedFieldValues, + FailedDocsDetails, UpdateFieldLimitResponse, } from '../../../common/api_types'; @@ -43,6 +46,10 @@ export interface DataStreamDetailsServiceStartDeps { export interface IDataStreamDetailsClient { getDataStreamSettings(params: GetDataStreamSettingsParams): Promise; getDataStreamDetails(params: GetDataStreamDetailsParams): Promise; + getFailedDocsDetails(params: GetDataStreamFailedDocsDetailsParams): Promise; + getFailedDocsErrors( + params: GetDataStreamFailedDocsErrorsParams + ): Promise<{ errors: Record }>; getDataStreamDegradedFields( params: GetDataStreamDegradedFieldsParams ): Promise; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 8e218819315b2..b00fd5cbcac2f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -11,8 +11,10 @@ import rison from '@kbn/rison'; import { KNOWN_TYPES } from '../../../common/constants'; import { DataStreamDegradedDocsResponse, + DataStreamFailedDocsResponse, DataStreamTotalDocsResponse, getDataStreamDegradedDocsResponseRt, + getDataStreamFailedDocsResponseRt, getDataStreamsStatsResponseRt, getDataStreamTotalDocsResponseRt, getIntegrationsResponseRt, @@ -23,6 +25,7 @@ import { import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, + GetDataStreamsFailedDocsStatsQuery, GetDataStreamsStatsQuery, GetDataStreamsStatsResponse, GetDataStreamsTotalDocsQuery, @@ -108,6 +111,30 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return degradedDocs; } + public async getDataStreamsFailedStats(params: GetDataStreamsFailedDocsStatsQuery) { + const types = params.types.length === 0 ? KNOWN_TYPES : params.types; + const response = await this.http + .get('/internal/dataset_quality/data_streams/failed_docs', { + query: { + ...params, + types: rison.encodeArray(types), + }, + }) + .catch((error) => { + throw new DatasetQualityError(`Failed to fetch data streams failed stats: ${error}`, error); + }); + + const { failedDocs } = decodeOrThrow( + getDataStreamFailedDocsResponseRt, + (message: string) => + new DatasetQualityError( + `Failed to decode data streams failed docs stats response: ${message}` + ) + )(response); + + return failedDocs; + } + public async getNonAggregatableDatasets(params: GetNonAggregatableDataStreamsParams) { const types = params.types.length === 0 ? KNOWN_TYPES : params.types; const response = await this.http diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts index 240e5519cfc3d..96da4b0d0c723 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts @@ -9,6 +9,7 @@ import { HttpStart } from '@kbn/core/public'; import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, + GetDataStreamsFailedDocsStatsQuery, GetDataStreamsStatsQuery, GetDataStreamsTotalDocsQuery, GetNonAggregatableDataStreamsParams, @@ -31,6 +32,9 @@ export interface IDataStreamsStatsClient { getDataStreamsDegradedStats( params?: GetDataStreamsDegradedDocsStatsQuery ): Promise; + getDataStreamsFailedStats( + params?: GetDataStreamsFailedDocsStatsQuery + ): Promise; getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery): Promise; getIntegrations(): Promise; getNonAggregatableDatasets( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index 7c77fe9d59422..d6d266c6753b8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -38,6 +38,7 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { }, dataStreamStats: [], degradedDocStats: [], + failedDocStats: [], totalDocsStats: DEFAULT_DICTIONARY_TYPE, filters: { inactive: true, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index 0dea80104245f..f798256f65149 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -47,3 +47,12 @@ export const fetchIntegrationsFailedNotifier = (toasts: IToasts, error: Error) = text: error.message, }); }; + +export const fetchFailedStatsFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchFailedStatsFailed', { + defaultMessage: "We couldn't get your failed docs information.", + }), + text: error.message, + }); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 1217e52894ce7..9b5049c3a6c30 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -24,6 +24,7 @@ import { DEFAULT_CONTEXT } from './defaults'; import { fetchDatasetStatsFailedNotifier, fetchDegradedStatsFailedNotifier, + fetchFailedStatsFailedNotifier, fetchIntegrationsFailedNotifier, fetchTotalDocsFailedNotifier, } from './notifications'; @@ -125,6 +126,41 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }, }, + failedDocs: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadFailedDocs', + onDone: { + target: 'loaded', + actions: ['storeFailedDocStats', 'storeDatasets'], + }, + onError: [ + { + target: 'unauthorized', + cond: 'checkIfActionForbidden', + }, + { + target: 'loaded', + actions: ['notifyFetchFailedStatsFailed'], + }, + ], + }, + }, + loaded: {}, + unauthorized: { type: 'final' }, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'failedDocs.fetching', + actions: ['storeTimeRange'], + }, + REFRESH_DATA: { + target: 'failedDocs.fetching', + }, + }, + }, docsStats: { initial: 'fetching', states: { @@ -381,6 +417,9 @@ export const createPureDatasetQualityControllerStateMachine = ( storeDegradedDocStats: assign((_context, event: DoneInvokeEvent) => ({ degradedDocStats: event.data, })), + storeFailedDocStats: assign((_context, event: DoneInvokeEvent) => ({ + failedDocStats: event.data, + })), storeNonAggregatableDatasets: assign( (_context, event: DoneInvokeEvent) => ({ nonAggregatableDatasets: event.data.datasets, @@ -404,6 +443,7 @@ export const createPureDatasetQualityControllerStateMachine = ( datasets: generateDatasets( context.dataStreamStats, context.degradedDocStats, + context.failedDocStats, context.integrations, context.totalDocsStats ), @@ -447,6 +487,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchIntegrationsFailedNotifier(toasts, event.data), notifyFetchTotalDocsFailed: (_context, event: DoneInvokeEvent, meta) => fetchTotalDocsFailedNotifier(toasts, event.data, meta), + notifyFetchFailedStatsFailed: (_context, event: DoneInvokeEvent) => + fetchFailedStatsFailedNotifier(toasts, event.data), }, services: { loadDataStreamStats: (context, _event) => @@ -489,6 +531,16 @@ export const createDatasetQualityControllerStateMachine = ({ end, }); }, + loadFailedDocs: (context) => { + const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); + + return dataStreamStatsClient.getDataStreamsFailedStats({ + types: context.filters.types as DataStreamType[], + datasetQuery: context.filters.query, + start, + end, + }); + }, loadNonAggregatableDatasets: (context) => { const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index de7fdbf9fbd77..9d71984ec51d0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -60,6 +60,10 @@ export interface WithDegradedDocs { degradedDocStats: DataStreamDocsStat[]; } +export interface WithFailedDocs { + failedDocStats: DataStreamDocsStat[]; +} + export interface WithNonAggregatableDatasets { nonAggregatableDatasets: string[]; } @@ -76,6 +80,7 @@ export type DefaultDatasetQualityControllerState = WithTableOptions & WithDataStreamStats & WithTotalDocs & WithDegradedDocs & + WithFailedDocs & WithDatasets & WithFilters & WithNonAggregatableDatasets & @@ -92,10 +97,18 @@ export type DatasetQualityControllerTypeState = value: 'stats.datasets.loaded'; context: DefaultDatasetQualityStateContext; } + | { + value: 'stats.docsStats.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'stats.degradedDocs.fetching'; context: DefaultDatasetQualityStateContext; } + | { + value: 'stats.failedDocs.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'stats.nonAggregatableDatasets.fetching'; context: DefaultDatasetQualityStateContext; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts index 26a51014b3abb..acb841964e027 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts @@ -11,7 +11,7 @@ import { DEFAULT_DEGRADED_FIELD_SORT_FIELD, DEFAULT_TIME_RANGE, } from '../../../common/constants'; -import { DefaultDatasetQualityDetailsContext } from './types'; +import { DefaultDatasetQualityDetailsContext, QualityIssueType } from './types'; export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = { degradedFields: { @@ -24,10 +24,21 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = { }, }, }, + failedDocsErrors: { + table: { + page: 0, + rowsPerPage: 10, + sort: { + field: 'type', + direction: DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + }, + }, + }, isIndexNotFoundError: false, timeRange: { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH, }, showCurrentQualityIssues: false, + qualityIssuesChart: 'degraded' as unknown as QualityIssueType, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 8ac65a7dca4a7..ac3b4fcd0502d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -24,7 +24,10 @@ import { DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, + FailedDocsDetails, + FailedDocsErrors, NonAggregatableDatasets, + QualityIssue, UpdateFieldLimitResponse, } from '../../../common/api_types'; import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications'; @@ -121,6 +124,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( '#DatasetQualityDetailsController.initializing.checkBreakdownFieldIsEcs.fetching', actions: ['storeBreakDownField'], }, + QUALITY_ISSUES_CHART_CHANGE: { + target: 'done', + actions: ['storeQualityIssuesChart'], + }, }, }, }, @@ -169,6 +176,37 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( loadingIntegrationsAndDegradedFields: { type: 'parallel', states: { + dataStreamFailedDocs: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadFailedDocsDetails', + onDone: { + target: 'done', + actions: ['storeFailedDocsDetails', 'raiseDegradedFieldsLoaded'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], + }, + }, + done: { + on: { + UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { + target: 'done', + actions: ['storeDegradedFieldTableOptions'], + }, + }, + }, + }, + }, dataStreamDegradedFields: { initial: 'fetching', states: { @@ -284,103 +322,155 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, open: { - initial: 'initialized', + initial: 'initializing', states: { - initialized: { - type: 'parallel', + initializing: { + always: [ + { + target: + '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open.degradedFieldFlyout', + cond: 'isDegradedFieldFlyout', + }, + { + target: + '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open.failedDocsFlyout', + }, + ], + }, + degradedFieldFlyout: { + initial: 'initialized', states: { - ignoredValues: { - initial: 'fetching', + initialized: { + type: 'parallel', states: { - fetching: { - invoke: { - src: 'loadDegradedFieldValues', - onDone: { - target: 'done', - actions: ['storeDegradedFieldValues'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', - }, - { - target: 'done', + ignoredValues: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFieldValues', + onDone: { + target: 'done', + actions: ['storeDegradedFieldValues'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], }, - ], - }, - }, - done: {}, - }, - }, - mitigation: { - initial: 'analyzing', - states: { - analyzing: { - invoke: { - src: 'analyzeDegradedField', - onDone: { - target: 'analyzed', - actions: ['storeDegradedFieldAnalysis'], - }, - onError: { - target: 'analyzed', }, + done: {}, }, }, - analyzed: { - on: { - SET_NEW_FIELD_LIMIT: { - target: 'mitigating', - actions: 'storeNewFieldLimit', + mitigation: { + initial: 'analyzing', + states: { + analyzing: { + invoke: { + src: 'analyzeDegradedField', + onDone: { + target: 'analyzed', + actions: ['storeDegradedFieldAnalysis'], + }, + onError: { + target: 'analyzed', + }, + }, }, - }, - }, - mitigating: { - invoke: { - src: 'saveNewFieldLimit', - onDone: [ - { - target: 'askingForRollover', - actions: 'storeNewFieldLimitResponse', - cond: 'hasFailedToUpdateLastBackingIndex', + analyzed: { + on: { + SET_NEW_FIELD_LIMIT: { + target: 'mitigating', + actions: 'storeNewFieldLimit', + }, }, - { - target: 'success', - actions: 'storeNewFieldLimitResponse', + }, + mitigating: { + invoke: { + src: 'saveNewFieldLimit', + onDone: [ + { + target: 'askingForRollover', + actions: 'storeNewFieldLimitResponse', + cond: 'hasFailedToUpdateLastBackingIndex', + }, + { + target: 'success', + actions: 'storeNewFieldLimitResponse', + }, + ], + onError: { + target: 'error', + actions: [ + 'storeNewFieldLimitErrorResponse', + 'notifySaveNewFieldLimitError', + ], + }, }, - ], - onError: { - target: 'error', - actions: [ - 'storeNewFieldLimitErrorResponse', - 'notifySaveNewFieldLimitError', - ], }, - }, - }, - askingForRollover: { - on: { - ROLLOVER_DATA_STREAM: { - target: 'rollingOver', + askingForRollover: { + on: { + ROLLOVER_DATA_STREAM: { + target: 'rollingOver', + }, + }, + }, + rollingOver: { + invoke: { + src: 'rolloverDataStream', + onDone: { + target: 'success', + actions: ['raiseForceTimeRangeRefresh'], + }, + onError: { + target: 'error', + actions: 'notifySaveNewFieldLimitError', + }, + }, }, + success: {}, + error: {}, }, }, - rollingOver: { - invoke: { - src: 'rolloverDataStream', - onDone: { - target: 'success', - actions: ['raiseForceTimeRangeRefresh'], - }, - onError: { - target: 'error', - actions: 'notifySaveNewFieldLimitError', + }, + }, + }, + }, + failedDocsFlyout: { + initial: 'initialized', + states: { + initialized: { + type: 'parallel', + states: { + failedDocsErrors: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadfailedDocsErrors', + onDone: { + target: 'done', + actions: ['storefailedDocsErrors'], + }, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', + }, + { + target: 'done', + }, + ], + }, }, + done: {}, }, }, - success: {}, - error: {}, }, }, }, @@ -450,6 +540,11 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( } : {}; }), + storeQualityIssuesChart: assign((_context, event) => { + return 'qualityIssuesChart' in event + ? { qualityIssuesChart: event.qualityIssuesChart } + : {}; + }), storeBreakDownField: assign((_context, event) => { return 'breakdownField' in event ? { breakdownField: event.breakdownField } : {}; }), @@ -460,12 +555,53 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( } : {}; }), + storeFailedDocsDetails: assign((context, event: DoneInvokeEvent) => { + return 'data' in event + ? { + degradedFields: { + ...context.degradedFields, + data: [ + ...(context.degradedFields.data ?? []).filter( + (field) => field.type !== 'failed' + ), + ...(event.data.timeSeries.length > 0 + ? [ + { + ...event.data, + name: 'failedDocs', + type: 'failed', + }, + ] + : []), + ], + }, + } + : {}; + }), + storefailedDocsErrors: assign((context, event: DoneInvokeEvent) => { + return 'data' in event + ? { + failedDocsErrors: { + ...context.failedDocsErrors, + data: event.data.errors, + }, + } + : {}; + }), storeDegradedFields: assign((context, event: DoneInvokeEvent) => { return 'data' in event ? { degradedFields: { ...context.degradedFields, - data: event.data.degradedFields, + data: [ + ...(context.degradedFields.data ?? []).filter( + (field) => field.type !== 'degraded' + ), + ...(event.data.degradedFields.map((field) => ({ + ...field, + type: 'degraded', + })) as QualityIssue[]), + ], }, } : {}; @@ -496,7 +632,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }), storeExpandedDegradedField: assign((_, event) => { return { - expandedDegradedField: 'fieldName' in event ? event.fieldName : undefined, + expandedQualityIssue: + 'qualityIssue' in event + ? { name: event.qualityIssue.name, type: event.qualityIssue.type } + : undefined, }; }), toggleCurrentQualityIssues: assign((context) => { @@ -583,16 +722,19 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, shouldOpenFlyout: (context) => { return ( - Boolean(context.expandedDegradedField) && + Boolean(context.expandedQualityIssue) && Boolean( context.degradedFields.data?.some( - (field) => field.name === context.expandedDegradedField + (field) => field.name === context.expandedQualityIssue?.name ) ) ); }, + isDegradedFieldFlyout: (context) => { + return Boolean(context.expandedQualityIssue?.type === 'degraded'); + }, hasNoDegradedFieldsSelected: (context) => { - return !Boolean(context.expandedDegradedField); + return !Boolean(context.expandedQualityIssue); }, hasFailedToUpdateLastBackingIndex: (_, event) => { return ( @@ -685,6 +827,15 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ return false; }, + loadFailedDocsDetails: (context) => { + const { startDate: start, endDate: end } = getDateISORange(context.timeRange); + + return dataStreamDetailsClient.getFailedDocsDetails({ + dataStream: context.dataStream, + start, + end, + }); + }, loadDegradedFields: (context) => { const { startDate: start, endDate: end } = getDateISORange(context.timeRange); @@ -706,10 +857,10 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ }, loadDegradedFieldValues: (context) => { - if ('expandedDegradedField' in context && context.expandedDegradedField) { + if ('expandedQualityIssue' in context && context.expandedQualityIssue) { return dataStreamDetailsClient.getDataStreamDegradedFieldValues({ dataStream: context.dataStream, - degradedField: context.expandedDegradedField, + degradedField: context.expandedQualityIssue.name, }); } return Promise.resolve(); @@ -717,19 +868,31 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ analyzeDegradedField: (context) => { if (context?.degradedFields?.data?.length) { const selectedDegradedField = context.degradedFields.data.find( - (field) => field.name === context.expandedDegradedField + (field) => field.name === context.expandedQualityIssue?.name ); - if (selectedDegradedField) { + if (selectedDegradedField && selectedDegradedField.type === 'degraded') { return dataStreamDetailsClient.analyzeDegradedField({ dataStream: context.dataStream, - degradedField: context.expandedDegradedField!, - lastBackingIndex: selectedDegradedField.indexFieldWasLastPresentIn, + degradedField: context.expandedQualityIssue?.name!, + lastBackingIndex: selectedDegradedField.indexFieldWasLastPresentIn!, }); } } return Promise.resolve(); }, + loadfailedDocsErrors: (context) => { + if ('expandedQualityIssue' in context && context.expandedQualityIssue) { + const { startDate: start, endDate: end } = getDateISORange(context.timeRange); + + return dataStreamDetailsClient.getFailedDocsErrors({ + dataStream: context.dataStream, + start, + end, + }); + } + return Promise.resolve(); + }, loadDataStreamSettings: (context) => { return dataStreamDetailsClient.getDataStreamSettings({ dataStream: context.dataStream, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index cdebcfbe53d86..90beece69886e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -12,16 +12,20 @@ import { DataStreamDetails, DataStreamRolloverResponse, DataStreamSettings, - DegradedField, DegradedFieldAnalysis, DegradedFieldResponse, DegradedFieldValues, + FailedDocsDetails, + FailedDocsErrors, NonAggregatableDatasets, + QualityIssue, UpdateFieldLimitResponse, } from '../../../common/api_types'; import { TableCriteria, TimeRangeConfig } from '../../../common/types'; import { Integration } from '../../../common/data_streams_stats/integration'; +export type QualityIssueType = QualityIssue['type']; + export interface DataStream { name: string; type: string; @@ -31,12 +35,12 @@ export interface DataStream { export interface DegradedFieldsTableConfig { table: TableCriteria; - data?: DegradedField[]; + data?: QualityIssue[]; } export interface DegradedFieldsWithData { table: TableCriteria; - data: DegradedField[]; + data: QualityIssue[]; } export interface FieldLimit { @@ -50,13 +54,18 @@ export interface WithDefaultControllerState { degradedFields: DegradedFieldsTableConfig; timeRange: TimeRangeConfig; showCurrentQualityIssues: boolean; + qualityIssuesChart: QualityIssueType; breakdownField?: string; isBreakdownFieldEcs?: boolean; isIndexNotFoundError?: boolean; integration?: Integration; - expandedDegradedField?: string; + expandedQualityIssue?: { + name: string; + type: QualityIssueType; + }; isNonAggregatable?: boolean; fieldLimit?: FieldLimit; + failedDocsErrors?: WithFailedDocsErrors['failedDocsErrors']; } export interface WithDataStreamDetails { @@ -88,6 +97,13 @@ export interface WithIntegration { integrationDashboards?: Dashboard[]; } +export interface WithFailedDocsErrors { + failedDocsErrors: { + table: TableCriteria; + data?: FailedDocsErrors['errors']; + }; +} + export interface WithDegradedFieldValues { degradedFieldValues: DegradedFieldValues; } @@ -108,7 +124,12 @@ export interface WithNewFieldLimitResponse { export type DefaultDatasetQualityDetailsContext = Pick< WithDefaultControllerState, - 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues' + | 'degradedFields' + | 'timeRange' + | 'isIndexNotFoundError' + | 'showCurrentQualityIssues' + | 'qualityIssuesChart' + | 'failedDocsErrors' >; export type DatasetQualityDetailsControllerTypeState = @@ -165,26 +186,26 @@ export type DatasetQualityDetailsControllerTypeState = } | { value: - | 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching' - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing'; + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.ignoredValues.fetching' + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.analyzing'; context: WithDefaultControllerState & WithDegradedFieldsData; } | { - value: 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.done'; + value: 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.ignoredValues.done'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues; } | { value: - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed' - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating' - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover' - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver' - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.success' - | 'initializing.degradedFieldFlyout.open.initialized.mitigation.error'; + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.analyzed' + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.mitigating' + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.askingForRollover' + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.rollingOver' + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.success' + | 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.error'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis; } | { - value: 'initializing.degradedFieldFlyout.open.initialized.mitigation.success'; + value: 'initializing.degradedFieldFlyout.open.initialized.degradedFieldFlyout.mitigation.success'; context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues & @@ -203,7 +224,10 @@ export type DatasetQualityDetailsControllerEvent = } | { type: 'OPEN_DEGRADED_FIELD_FLYOUT'; - fieldName: string | undefined; + qualityIssue: { + name: string; + type: QualityIssueType; + }; } | { type: 'CLOSE_DEGRADED_FIELD_FLYOUT'; @@ -211,6 +235,10 @@ export type DatasetQualityDetailsControllerEvent = | { type: 'DEGRADED_FIELDS_LOADED'; } + | { + type: 'QUALITY_ISSUES_CHART_CHANGE'; + qualityIssuesChart: QualityIssueType; + } | { type: 'BREAKDOWN_FIELD_CHANGE'; breakdownField: string | undefined; @@ -230,6 +258,7 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts new file mode 100644 index 0000000000000..a320d5d4641d7 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) { + return totalDocs && count ? Number(((count / totalDocs) * 100).toFixed(2)) : 0; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index b75c74c2fd728..135979e63fb60 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -73,18 +73,27 @@ describe('generateDatasets', () => { }; const degradedDocs = [ - { - dataset: 'logs-system.application-default', - count: 0, - }, { dataset: 'logs-synth-default', count: 6, }, ]; - it('merges integrations information with dataStreamStats and degradedDocs', () => { - const datasets = generateDatasets(dataStreamStats, degradedDocs, integrations, totalDocs); + const failedDocs = [ + { + dataset: 'logs-system.application-default', + count: 2, + }, + ]; + + it('merges integrations information with dataStreamStats, degradedDocs and failedDocs', () => { + const datasets = generateDatasets( + dataStreamStats, + degradedDocs, + failedDocs, + integrations, + totalDocs + ); expect(datasets).toEqual([ { @@ -101,12 +110,16 @@ describe('generateDatasets', () => { userPrivileges: { canMonitor: true, }, - docsInTimeRange: 100, - quality: 'good', + docsInTimeRange: 102, + quality: 'degraded', degradedDocs: { percentage: 0, count: 0, }, + failedDocs: { + percentage: 1.96, + count: 2, + }, }, { name: 'synth', @@ -128,6 +141,10 @@ describe('generateDatasets', () => { count: 6, percentage: 6, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); @@ -136,6 +153,7 @@ describe('generateDatasets', () => { const datasets = generateDatasets( dataStreamStats, degradedDocs, + failedDocs, integrations, DEFAULT_DICTIONARY_TYPE ); @@ -155,12 +173,16 @@ describe('generateDatasets', () => { userPrivileges: { canMonitor: true, }, - docsInTimeRange: 0, - quality: 'good', + docsInTimeRange: 2, + quality: 'poor', degradedDocs: { percentage: 0, count: 0, }, + failedDocs: { + percentage: 100, + count: 2, + }, }, { name: 'synth', @@ -182,12 +204,16 @@ describe('generateDatasets', () => { count: 6, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); it('merges integrations information with degradedDocs', () => { - const datasets = generateDatasets([], degradedDocs, integrations, totalDocs); + const datasets = generateDatasets([], degradedDocs, [], integrations, totalDocs); expect(datasets).toEqual([ { @@ -208,6 +234,10 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'synth', @@ -227,12 +257,16 @@ describe('generateDatasets', () => { count: 6, percentage: 6, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); it('merges integrations information with degradedDocs and totalDocs', () => { - const datasets = generateDatasets([], degradedDocs, integrations, { + const datasets = generateDatasets([], degradedDocs, [], integrations, { ...totalDocs, logs: [...totalDocs.logs, { dataset: 'logs-another-default', count: 100 }], }); @@ -256,6 +290,10 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'synth', @@ -275,6 +313,10 @@ describe('generateDatasets', () => { count: 6, percentage: 6, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'another', @@ -294,12 +336,16 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); it('merges integrations information with dataStreamStats', () => { - const datasets = generateDatasets(dataStreamStats, [], integrations, totalDocs); + const datasets = generateDatasets(dataStreamStats, [], [], integrations, totalDocs); expect(datasets).toEqual([ { @@ -322,6 +368,10 @@ describe('generateDatasets', () => { count: 0, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'synth', @@ -343,6 +393,10 @@ describe('generateDatasets', () => { count: 0, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); @@ -360,7 +414,7 @@ describe('generateDatasets', () => { }, }; - const datasets = generateDatasets([nonDefaultDataset], [], integrations, totalDocs); + const datasets = generateDatasets([nonDefaultDataset], [], [], integrations, totalDocs); expect(datasets).toEqual([ { @@ -383,6 +437,10 @@ describe('generateDatasets', () => { count: 0, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 8e9f2f3db7083..82eb41f5b1651 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { DEFAULT_DEGRADED_DOCS } from '../../common/constants'; +import { DEFAULT_QUALITY_DOC_STATS } from '../../common/constants'; import { DataStreamDocsStat } from '../../common/api_types'; import { DataStreamStatType } from '../../common/data_streams_stats/types'; import { mapPercentageToQuality } from '../../common/utils'; @@ -13,9 +13,12 @@ import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { DictionaryType } from '../state_machines/dataset_quality_controller/src/types'; import { flattenStats } from './flatten_stats'; +import { calculatePercentage } from './calculate_percentage'; + export function generateDatasets( dataStreamStats: DataStreamStatType[] = [], degradedDocStats: DataStreamDocsStat[] = [], + failedDocStats: DataStreamDocsStat[] = [], integrations: Integration[], totalDocsStats: DictionaryType ): DataStreamStat[] { @@ -51,6 +54,26 @@ export function generateDatasets( const totalDocsMap: Record = Object.fromEntries(totalDocs.map(({ dataset, count }) => [dataset, count])); + const failedMap: Record< + DataStreamDocsStat['dataset'], + { + percentage: number; + count: DataStreamDocsStat['count']; + } + > = failedDocStats.reduce( + (failedMapAcc, { dataset, count }) => + Object.assign(failedMapAcc, { + [dataset]: { + count, + percentage: calculatePercentage({ + totalDocs: (totalDocsMap[dataset] ?? 0) + count, + count, + }), + }, + }), + {} + ); + const degradedMap: Record< DataStreamDocsStat['dataset'], { @@ -62,8 +85,8 @@ export function generateDatasets( Object.assign(degradedMapAcc, { [dataset]: { count, - percentage: DataStreamStat.calculatePercentage({ - totalDocs: totalDocsMap[dataset], + percentage: calculatePercentage({ + totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0), count, }), }, @@ -72,19 +95,31 @@ export function generateDatasets( ); if (!dataStreamStats.length) { - // We want to pick up all datasets even when they don't have degraded docs - const dataStreams = [...new Set([...Object.keys(totalDocsMap), ...Object.keys(degradedMap)])]; + // We want to pick up all datasets even when they don't have degraded docs or failed docs + const dataStreams = [ + ...new Set([ + ...Object.keys(totalDocsMap), + ...Object.keys(degradedMap), + ...Object.keys(failedMap), + ]), + ]; return dataStreams.map((dataset) => - DataStreamStat.fromDegradedDocStat({ - degradedDocStat: { dataset, ...(degradedMap[dataset] || DEFAULT_DEGRADED_DOCS) }, + DataStreamStat.fromQualityStats({ + datasetName: dataset, + degradedDocStat: degradedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, + failedDocStat: failedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, datasetIntegrationMap, - totalDocs: totalDocsMap[dataset] ?? 0, + totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0), }) ); } return dataStreamStats?.map((dataStream) => { const dataset = DataStreamStat.create(dataStream); + const qualityStats = [ + (degradedMap[dataset.rawName] || dataset.degradedDocs).percentage, + (failedMap[dataset.rawName] || dataset.failedDocs).percentage, + ]; return { ...dataset, @@ -93,10 +128,10 @@ export function generateDatasets( datasetIntegrationMap[dataset.name]?.integration ?? integrationsMap[dataStream.integration ?? ''], degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs, - docsInTimeRange: totalDocsMap[dataset.rawName] ?? 0, - quality: mapPercentageToQuality( - (degradedMap[dataset.rawName] || dataset.degradedDocs).percentage - ), + failedDocs: failedMap[dataset.rawName] || dataset.failedDocs, + docsInTimeRange: + (totalDocsMap[dataset.rawName] ?? 0) + (failedMap[dataset.rawName]?.count ?? 0), + quality: mapPercentageToQuality(qualityStats), }; }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts index 3185367e39aca..fe70b6e737a27 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +export * from './calculate_percentage'; export * from './filter_inactive_datasets'; export * from './generate_datasets'; export * from './use_kibana'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 288eff11b92a8..eca6f7c7f3d71 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -21,6 +21,7 @@ import { createDatasetQualityESClient } from '../../../utils'; import { dataStreamService, datasetQualityPrivileges } from '../../../services'; import { getDataStreams } from '../get_data_streams'; import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats'; +import { getFailedDocsPaginated } from '../get_failed_docs/get_failed_docs'; export async function getDataStreamSettings({ esClient, @@ -92,6 +93,14 @@ export async function getDataStreamDetails({ end ); + const failedDocs = await getFailedDocsPaginated({ + esClient: esClientAsCurrentUser, + types: [], + datasetQuery: dataStream, + start, + end, + }); + const avgDocSizeInBytes = hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0 ? isServerless @@ -103,6 +112,7 @@ export async function getDataStreamDetails({ return { ...dataStreamSummaryStats, + failedDocsCount: failedDocs[0]?.count, sizeBytes, lastActivity: esDataStream?.lastActivity, userPrivileges: { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts index 0bb0b6a695fef..b8e3cf60a8b31 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts @@ -11,7 +11,7 @@ import { DegradedFieldResponse } from '../../../../common/api_types'; import { MAX_DEGRADED_FIELDS } from '../../../../common/constants'; import { createDatasetQualityESClient } from '../../../utils'; import { _IGNORED, INDEX, TIMESTAMP } from '../../../../common/es_fields'; -import { getFieldIntervalInSeconds } from './get_interval'; +import { getFieldIntervalInSeconds } from '../../../utils/get_interval'; export async function getDegradedFields({ esClient, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs.ts new file mode 100644 index 0000000000000..896b6aa3daca5 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { rangeQuery } from '@kbn/observability-plugin/server'; +import { + extractIndexNameFromBackingIndex, + streamPartsToIndexPattern, +} from '../../../../common/utils'; +import { DataStreamType } from '../../../../common/types'; +import { DataStreamDocsStat } from '../../../../common/api_types'; +import { createDatasetQualityESClient } from '../../../utils'; +import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client'; + +const SIZE_LIMIT = 10000; + +async function getPaginatedResults(options: { + datasetQualityESClient: DatasetQualityESClient; + index: string; + start: number; + end: number; + after?: { dataset: string }; + prevResults?: Record; +}) { + const { datasetQualityESClient, index, start, end, after, prevResults = {} } = options; + + const bool = { + filter: [...rangeQuery(start, end)], + }; + + // TODO: Fix index for accesing failure store (::failures) and remove the search parameter + const response = await datasetQualityESClient.search({ + index, + size: 0, + query: { + bool, + }, + aggs: { + datasets: { + composite: { + ...(after ? { after } : {}), + size: SIZE_LIMIT, + sources: [{ dataset: { terms: { field: '_index' } } }], + }, + }, + }, + failure_store: 'only', + }); + + const currResults = (response.aggregations?.datasets.buckets ?? []).reduce((acc, curr) => { + const datasetName = extractIndexNameFromBackingIndex(curr.key.dataset as string); + + return { + ...acc, + [datasetName]: (acc[datasetName] ?? 0) + curr.doc_count, + }; + }, {} as Record); + + const results = { + ...prevResults, + ...currResults, + }; + + if ( + response.aggregations?.datasets.after_key && + response.aggregations?.datasets.buckets.length === SIZE_LIMIT + ) { + return getPaginatedResults({ + datasetQualityESClient, + index, + start, + end, + after: + (response.aggregations?.datasets.after_key as { + dataset: string; + }) || after, + prevResults: results, + }); + } + + return results; +} + +export async function getFailedDocsPaginated(options: { + esClient: ElasticsearchClient; + types: DataStreamType[]; + datasetQuery?: string; + start: number; + end: number; +}): Promise { + const { esClient, types, datasetQuery, start, end } = options; + + const datasetNames = datasetQuery + ? [datasetQuery] + : types.map((type) => + streamPartsToIndexPattern({ + typePattern: type, + datasetPattern: '*-*', + }) + ); + + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const datasets = await getPaginatedResults({ + datasetQualityESClient, + index: datasetNames.join(','), + start, + end, + }); + + return Object.entries(datasets).map(([dataset, count]) => ({ + dataset, + count, + })); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts new file mode 100644 index 0000000000000..928fecc8e3096 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_details.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { rangeQuery } from '@kbn/observability-plugin/server'; +import { TIMESTAMP } from '../../../../common/es_fields'; +import { FailedDocsDetails } from '../../../../common/api_types'; +import { getFieldIntervalInSeconds } from '../../../utils/get_interval'; +import { createDatasetQualityESClient } from '../../../utils'; + +export async function getFailedDocsDetails({ + esClient, + start, + end, + dataStream, +}: { + esClient: ElasticsearchClient; + start: number; + end: number; + dataStream: string; +}): Promise { + const fieldInterval = getFieldIntervalInSeconds({ start, end }); + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const filterQuery = [...rangeQuery(start, end)]; + + const aggs = { + lastOccurrence: { + max: { + field: TIMESTAMP, + }, + }, + timeSeries: { + date_histogram: { + field: TIMESTAMP, + fixed_interval: `${fieldInterval}s`, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, + }, + }, + }, + }; + + const response = await datasetQualityESClient.search({ + index: dataStream, + track_total_hits: true, + size: 0, + query: { + bool: { + filter: filterQuery, + }, + }, + aggs, + failure_store: 'only', + }); + + return { + count: response.hits.total.value, + lastOccurrence: response.aggregations?.lastOccurrence.value, + timeSeries: + response.aggregations?.timeSeries.buckets.map((timeSeriesBucket) => ({ + x: timeSeriesBucket.key, + y: timeSeriesBucket.doc_count, + })) ?? [], + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts new file mode 100644 index 0000000000000..c8fd640b4cd21 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/get_failed_docs_errors.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { SearchHit } from '@kbn/es-types'; +import { rangeQuery } from '@kbn/observability-plugin/server'; +import { createDatasetQualityESClient } from '../../../utils'; +import { TIMESTAMP } from '../../../../common/es_fields'; + +export async function getFailedDocsErrors({ + esClient, + start, + end, + dataStream, +}: { + esClient: ElasticsearchClient; + start: number; + end: number; + dataStream: string; +}): Promise<{ errors: Array<{ type: string; message: string }> }> { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const bool = { + filter: [...rangeQuery(start, end)], + }; + + const response = await datasetQualityESClient.search({ + index: dataStream, + size: 10000, + query: { + bool, + }, + sort: [ + { + [TIMESTAMP]: { + order: 'desc', + }, + }, + ], + failure_store: 'only', + }); + + const errors = extractAndDeduplicateValues(response.hits.hits); + + return { + errors, + }; +} + +function extractAndDeduplicateValues( + searchHits: SearchHit[] +): Array<{ type: string; message: string }> { + const values: Record = {}; + + searchHits.forEach((hit: any) => { + const fieldKey = hit._source?.error?.type; + const fieldValue = hit._source?.error?.message; + if (values[fieldKey]) { + values[fieldKey].push(fieldValue); + } else { + values[fieldKey] = [fieldValue]; + } + }); + + Object.keys(values).forEach((key) => { + values[key] = Array.from(new Set(values[key])); + }); + + return Object.entries(values) + .map(([key, messages]) => messages.map((message) => ({ type: key, message }))) + .flat(); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/routes.ts new file mode 100644 index 0000000000000..88c4a49515dec --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs/routes.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { DataStreamDocsStat, FailedDocsDetails } from '../../../../common/api_types'; +import { rangeRt, typesRt } from '../../../types/default_api_types'; +import { createDatasetQualityServerRoute } from '../../create_datasets_quality_server_route'; +import { getFailedDocsPaginated } from './get_failed_docs'; +import { getFailedDocsDetails } from './get_failed_docs_details'; +import { getFailedDocsErrors } from './get_failed_docs_errors'; + +const failedDocsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/failed_docs', + params: t.type({ + query: t.intersection([ + rangeRt, + t.type({ types: typesRt }), + t.partial({ + datasetQuery: t.string, + }), + ]), + }), + options: { + tags: [], + }, + async handler(resources): Promise<{ + failedDocs: DataStreamDocsStat[]; + }> { + const { context, params } = resources; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const failedDocs = await getFailedDocsPaginated({ + esClient, + ...params.query, + }); + + return { + failedDocs, + }; + }, +}); + +const failedDocsDetailsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + query: rangeRt, + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const { dataStream } = params.path; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + return await getFailedDocsDetails({ + esClient, + dataStream, + ...params.query, + }); + }, +}); + +const failedDocsErrorsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs/errors', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + query: rangeRt, + }), + options: { + tags: [], + }, + async handler(resources): Promise<{ errors: Array<{ type: string; message: string }> }> { + const { context, params } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + return await getFailedDocsErrors({ + esClient, + dataStream: params.path.dataStream, + ...params.query, + }); + }, +}); + +export const failedDocsRouteRepository = { + ...failedDocsRoute, + ...failedDocsDetailsRoute, + ...failedDocsErrorsRoute, +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 3a60f0b9a8ef3..d2db5532681f5 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -8,32 +8,33 @@ import * as t from 'io-ts'; import { DataStreamDetails, + DataStreamDocsStat, + DataStreamRolloverResponse, DataStreamSettings, DataStreamStat, - NonAggregatableDatasets, - DegradedFieldResponse, DatasetUserPrivileges, - DegradedFieldValues, DegradedFieldAnalysis, - DataStreamDocsStat, + DegradedFieldResponse, + DegradedFieldValues, + NonAggregatableDatasets, UpdateFieldLimitResponse, - DataStreamRolloverResponse, } from '../../../common/api_types'; +import { datasetQualityPrivileges } from '../../services'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; +import { createDatasetQualityESClient } from '../../utils'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; -import { datasetQualityPrivileges } from '../../services'; import { getDataStreamDetails, getDataStreamSettings } from './get_data_stream_details'; import { getDataStreams } from './get_data_streams'; +import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; import { getDataStreamsStats } from './get_data_streams_stats'; +import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; import { getDegradedDocsPaginated } from './get_degraded_docs'; -import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; -import { getDegradedFields } from './get_degraded_fields'; -import { getDegradedFieldValues } from './get_degraded_field_values'; import { analyzeDegradedField } from './get_degraded_field_analysis'; -import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; -import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; +import { getDegradedFieldValues } from './get_degraded_field_values'; +import { getDegradedFields } from './get_degraded_fields'; +import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; import { updateFieldLimit } from './update_field_limit'; -import { createDatasetQualityESClient } from '../../utils'; +import { failedDocsRouteRepository } from './get_failed_docs/routes'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -421,4 +422,5 @@ export const dataStreamsRouteRepository = { ...analyzeDegradedFieldRoute, ...updateFieldLimitRoute, ...rolloverDataStream, + ...failedDocsRouteRepository, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 8a78b4163da95..53db8217d18d6 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -22,6 +22,8 @@ import { type DatasetQualityESSearchParams = ESSearchRequest & { size: number; + // TODO: Remove search parameter once ::failures is supported + failure_store?: 'only'; }; export type DatasetQualityESClient = ReturnType; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/get_interval.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/get_interval.ts similarity index 100% rename from x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/get_interval.ts rename to x-pack/plugins/observability_solution/dataset_quality/server/utils/get_interval.ts diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 970d4897b9e6a..6717c8ac84b6c 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -15292,18 +15292,14 @@ "xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip": "Le nombre de champs dans cet index a dépassé la limite maximale autorisée.", "xpack.datasetQuality.details.degradedField.cause.fieldMalformed": "champ mal formé", "xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip": "Le type de données du champ n'est pas défini correctement.", - "xpack.datasetQuality.details.degradedField.count": "Nombre de documents", "xpack.datasetQuality.details.degradedField.currentFieldLimit": "Limite de champ", - "xpack.datasetQuality.details.degradedField.field": "Champ", "xpack.datasetQuality.details.degradedField.fieldIgnored": "champ ignoré", - "xpack.datasetQuality.details.degradedField.lastOccurrence": "Dernière occurrence", "xpack.datasetQuality.details.degradedField.maximumCharacterLimit": "Longueur maximale des caractères", "xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex": "Ce problème a été détecté dans une ancienne version de l'ensemble de données, mais pas dans la version la plus récente.", "xpack.datasetQuality.details.degradedField.potentialCause": "Cause potentielle", "xpack.datasetQuality.details.degradedField.values": "Valeurs", "xpack.datasetQuality.details.degradedFieldsSectionTooltip": "Une liste partielle des problèmes de qualité détectés dans votre ensemble de données.", "xpack.datasetQuality.details.degradedFieldsTableLoadingText": "Chargement des champs dégradés", - "xpack.datasetQuality.details.degradedFieldsTableNoData": "Aucun champ dégradé n’a été trouvé", "xpack.datasetQuality.details.degradedFieldTable.collapseLabel": "Réduire", "xpack.datasetQuality.details.degradedFieldTable.expandLabel": "Développer", "xpack.datasetQuality.details.degradedFieldToggleSwitch": "Problèmes de qualité actuels uniquement", @@ -15344,7 +15340,6 @@ "xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "Nous n'avons pas pu obtenir d'informations sur les ensembles de données non agrégés.", "xpack.datasetQuality.fewDegradedDocsTooltip": "{degradedDocsCount} documents dégradés dans cet ensemble de données.", "xpack.datasetQuality.filterBar.placeholder": "Filtrer les ensembles de données", - "xpack.datasetQuality.flyout.degradedDocsTitle": "Documents dégradés", "xpack.datasetQuality.flyout.nonAggregatable.description": "{description}", "xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "{rolloverLink} manuellement cet ensemble de données pour empêcher des délais à l'avenir.", "xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset} est incompatible avec l'agrégation _ignored, ce qui peut entraîner des délais lors de la recherche de données. {howToFixIt}", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 1ae2f8380600d..3afbe2c09361f 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -15271,18 +15271,14 @@ "xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip": "このインデックスのフィールド数が許可された最大上限を超えています。", "xpack.datasetQuality.details.degradedField.cause.fieldMalformed": "フィールドの形式が正しくありません", "xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip": "フィールドのデータ型が正しく設定されていません。", - "xpack.datasetQuality.details.degradedField.count": "ドキュメント数", "xpack.datasetQuality.details.degradedField.currentFieldLimit": "フィールド上限", - "xpack.datasetQuality.details.degradedField.field": "フィールド", "xpack.datasetQuality.details.degradedField.fieldIgnored": "無視されたフィールド", - "xpack.datasetQuality.details.degradedField.lastOccurrence": "前回の発生", "xpack.datasetQuality.details.degradedField.maximumCharacterLimit": "最大文字長", "xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex": "この問題は、古いバージョンのデータセットでは検出されてましたが、最新のバージョンでは検出されませんでした。", "xpack.datasetQuality.details.degradedField.potentialCause": "潜在的な原因", "xpack.datasetQuality.details.degradedField.values": "値", "xpack.datasetQuality.details.degradedFieldsSectionTooltip": "データセットで見つかった品質の問題の部分的なリスト。", "xpack.datasetQuality.details.degradedFieldsTableLoadingText": "劣化したフィールドを読み込み中", - "xpack.datasetQuality.details.degradedFieldsTableNoData": "劣化したフィールドが見つかりません", "xpack.datasetQuality.details.degradedFieldTable.collapseLabel": "縮小", "xpack.datasetQuality.details.degradedFieldTable.expandLabel": "拡張", "xpack.datasetQuality.details.degradedFieldToggleSwitch": "現在の品質の問題のみ", @@ -15323,7 +15319,6 @@ "xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "集約可能なデータセット情報以外を取得できませんでした。", "xpack.datasetQuality.fewDegradedDocsTooltip": "このデータセットの{degradedDocsCount}個の劣化したドキュメント。", "xpack.datasetQuality.filterBar.placeholder": "データセットのフィルタリング", - "xpack.datasetQuality.flyout.degradedDocsTitle": "劣化したドキュメント", "xpack.datasetQuality.flyout.nonAggregatable.description": "{description}", "xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "今後の遅れを防止するには、手動でこのデータを{rolloverLink}してください。", "xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset}は_ignored集約をサポートしていません。データのクエリを実行するときに遅延が生じる可能性があります。{howToFixIt}", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 27eb8d93e9919..8ec4b7ce1f87a 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14965,18 +14965,14 @@ "xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip": "此索引中的字段数已超出允许的最大限制。", "xpack.datasetQuality.details.degradedField.cause.fieldMalformed": "字段格式不正确", "xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip": "未正确设置此字段的数据类型。", - "xpack.datasetQuality.details.degradedField.count": "文档计数", "xpack.datasetQuality.details.degradedField.currentFieldLimit": "字段限制", - "xpack.datasetQuality.details.degradedField.field": "字段", "xpack.datasetQuality.details.degradedField.fieldIgnored": "字段已忽略", - "xpack.datasetQuality.details.degradedField.lastOccurrence": "最后一次发生", "xpack.datasetQuality.details.degradedField.maximumCharacterLimit": "最大字符长度", "xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex": "在较早版本而不是最新版本的数据集中检测到此问题。", "xpack.datasetQuality.details.degradedField.potentialCause": "潜在原因", "xpack.datasetQuality.details.degradedField.values": "值", "xpack.datasetQuality.details.degradedFieldsSectionTooltip": "在数据集中发现的质量问题的部分列表。", "xpack.datasetQuality.details.degradedFieldsTableLoadingText": "正在加载已降级字段", - "xpack.datasetQuality.details.degradedFieldsTableNoData": "找不到已降级字段", "xpack.datasetQuality.details.degradedFieldTable.collapseLabel": "折叠", "xpack.datasetQuality.details.degradedFieldTable.expandLabel": "展开", "xpack.datasetQuality.details.degradedFieldToggleSwitch": "仅限当前的质量问题", @@ -15017,7 +15013,6 @@ "xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "无法获取非可聚合数据集信息。", "xpack.datasetQuality.fewDegradedDocsTooltip": "此数据集中的 {degradedDocsCount} 个已降级文档。", "xpack.datasetQuality.filterBar.placeholder": "筛选数据集", - "xpack.datasetQuality.flyout.degradedDocsTitle": "已降级文档", "xpack.datasetQuality.flyout.nonAggregatable.description": "{description}", "xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "手动 {rolloverLink} 此数据集以防止未来出现延迟。", "xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset} 不支持 _ignored 聚合,在查询数据时可能会导致延迟。{howToFixIt}", diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts new file mode 100644 index 0000000000000..1c711b447876b --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { APIClientRequestParamsOf } from '@kbn/dataset-quality-plugin/common/rest'; +import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { merge } from 'lodash'; +import rison from '@kbn/rison'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('synthtrace'); + const retry = getService('retry'); + const from = '2024-09-20T11:00:00.000Z'; + const to = '2024-09-20T11:01:00.000Z'; + const dataStreamType = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const syntheticsDataStreamName = `${dataStreamType}-${syntheticsDataset}-${namespace}`; + + const endpoint = 'GET /internal/dataset_quality/data_streams/failed_docs'; + type ApiParams = APIClientRequestParamsOf['params']['query']; + type DataStreamType = ApiParams['types'][0]; + + const processors = [ + { + script: { + tag: 'normalize log level', + lang: 'painless', + source: ` + String level = ctx['log.level']; + if ('0'.equals(level)) { + ctx['log.level'] = 'info'; + } else if ('1'.equals(level)) { + ctx['log.level'] = 'debug'; + } else if ('2'.equals(level)) { + ctx['log.level'] = 'warning'; + } else if ('3'.equals(level)) { + ctx['log.level'] = 'error'; + } else { + throw new Exception("Not a valid log level"); + } + `, + }, + }, + ]; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { types = rison.encodeArray(['logs']), start, end }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: Omit & { types?: DataStreamType[] }; + }) { + return roleScopedSupertestWithCookieCredentials + .get(`/internal/dataset_quality/data_streams/failed_docs`) + .query({ + types, + start, + end, + }); + } + + describe('DataStream failed docs', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + let synthtraceLogsEsClient: LogsSynthtraceEsClient; + + before(async () => { + synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient(); + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + + await synthtraceLogsEsClient.createCustomPipeline(processors); + await synthtraceLogsEsClient.updateIndexTemplate('logs', (template) => { + const next = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + }); + await synthtraceLogsEsClient.index([ + timerange(from, to) + .interval('1m') + .rate(1) + .generator((timestamp) => [ + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .logLevel('5') + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .logLevel('0') + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + ]), + ]); + }); + + after(async () => { + await synthtraceLogsEsClient.clean(); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('returns number of failed documents per DataStream', async () => { + await retry.tryForTime(180 * 1000, async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: from, + end: new Date().toISOString(), + }, + }); + + expect(resp.body.failedDocs.length).to.be(1); + expect(resp.body.failedDocs[0].dataset).to.be(syntheticsDataStreamName); + expect(resp.body.failedDocs[0].count).to.be(1); + }); + }); + + it('returns empty when all documents are outside timeRange', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: '2024-09-21T11:00:00.000Z', + end: '2024-09-21T11:01:00.000Z', + }, + }); + + expect(resp.body.failedDocs.length).to.be(0); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0481e882aee6e..5f9ce787a44dd 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -19,5 +19,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./degraded_fields')); loadTestFile(require.resolve('./data_stream_details')); loadTestFile(require.resolve('./degraded_field_values')); + loadTestFile(require.resolve('./failed_docs')); }); } diff --git a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts index 3692a17709a2e..17bb7aed54c81 100644 --- a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts +++ b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts @@ -118,7 +118,7 @@ export function createLogRecord( .create() .dataset(dataset) .message(msg.message) - .logLevel(isMalformed ? MORE_THAN_1024_CHARS : msg.level) + .logLevel(msg.level) .service(serviceName) .namespace(namespace) .defaults({ @@ -163,7 +163,7 @@ export function createDegradedFieldsRecord({ .create() .dataset(dataset) .message(MESSAGE_LOG_LEVELS[0].message) - .logLevel(MORE_THAN_1024_CHARS) + .logLevel(MESSAGE_LOG_LEVELS[0].level) .service(SERVICE_NAMES[0]) .namespace(defaultNamespace) .defaults({ @@ -189,7 +189,68 @@ export function createDegradedFieldsRecord({ }); } -export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; +/* +The helper function generates Failed Docs for the given dataset. + */ +export function createFailedRecords({ + to, + count = 1, + dataset, + namespace, + rate = 1, // rate of failed logs (min value 0, max value 1) +}: { + to: string; + count?: number; + dataset: string; + namespace?: string; + rate?: number; +}) { + return timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(count) + .fill(0) + .flatMap((_, index) => { + const isFailed = index % (count * rate) === 0; + return log + .create() + .dataset(dataset) + .message(MESSAGE_LOG_LEVELS[0].message) + .logLevel(isFailed ? 'anyLevel' : LogLevel.INFO) + .service(SERVICE_NAMES[0]) + .namespace(namespace ?? defaultNamespace) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + }) + .timestamp(timestamp); + }); + }); +} + +export const customLogLevelProcessor = [ + { + script: { + tag: 'normalize log level', + lang: 'painless', + source: ` + String level = ctx['log.level']; + if ('info'.equals(level)) { + ctx['log.level'] = 'info'; + } else if ('debug'.equals(level)) { + ctx['log.level'] = 'debug'; + } else if ('error'.equals(level)) { + ctx['log.level'] = 'error'; + } else { + throw new Exception("Not a valid log level"); + } + `, + }, + }, +]; + +export const datasetNames = ['synth.1', 'synth.2', 'synth.3', 'synth.failed']; export const defaultNamespace = 'default'; export const productionNamespace = 'production'; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts index 01dbbe1a0fc80..0fdd4a0681148 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts @@ -6,9 +6,12 @@ */ import expect from '@kbn/expect'; +import merge from 'lodash/merge'; import { DatasetQualityFtrProviderContext } from './config'; import { createDegradedFieldsRecord, + createFailedRecords, + customLogLevelProcessor, datasetNames, defaultNamespace, getInitialTestLogs, @@ -54,6 +57,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const regularDataStreamName = `logs-${datasetNames[0]}-${defaultNamespace}`; const degradedDatasetName = datasetNames[2]; const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const failedDatasetName = datasetNames[1]; + const failedDataStreamName = `logs-${failedDatasetName}-${defaultNamespace}`; describe('Dataset Quality Details', () => { before(async () => { @@ -63,6 +68,25 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Install Bitbucket Integration (package which does not has Dashboards) and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(bitbucketPkg); + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor, 'logs-apache.access@custom'); + await synthtrace.createComponentTemplate('logs-apache.access@custom', undefined, { + 'index.default_pipeline': 'logs-apache.access@custom', + }); + await synthtrace.updateIndexTemplate( + 'logs-apache.access', + (template: Record): Record => { + const next: Record = { + name: 'logs-apache.access', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + await synthtrace.index([ // Ingest basic logs getInitialTestLogs({ to, count: 4 }), @@ -87,6 +111,14 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid namespace: productionNamespace, isMalformed: true, }), + // Index failed docs for Apache integration + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: apacheAccessDatasetName, + namespace: productionNamespace, + rate: 0.5, + }), // Index logs for Bitbucket integration getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName }), ]); @@ -160,6 +192,19 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expect(currentUrl).to.not.contain('breakdownField'); }); }); + + it('reflects the selected quality issue chart state in url', async () => { + await PageObjects.datasetQuality.navigateToDetails({ dataStream: failedDataStreamName }); + + const charType = 'failedDocs'; + await PageObjects.datasetQuality.selectQualityIssuesChartType(charType); + + // Wait for URL to contain "qualityIssuesChart:failedDocs" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(decodeURIComponent(currentUrl)).to.contain(`qualityIssuesChart:${charType}`); + }); + }); }); describe('overview summary panel', () => { @@ -168,13 +213,14 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid dataStream: apacheAccessDataStreamName, }); - const { docsCountTotal, degradedDocs, services, hosts, size } = + const { docsCountTotal, degradedDocs, failedDocs, services, hosts, size } = await PageObjects.datasetQuality.parseOverviewSummaryPanelKpis(); - expect(parseInt(docsCountTotal, 10)).to.be(226); + expect(parseInt(docsCountTotal, 10)).to.be(306); expect(parseInt(degradedDocs, 10)).to.be(1); expect(parseInt(services, 10)).to.be(3); expect(parseInt(hosts, 10)).to.be(52); expect(parseInt(size, 10)).to.be.greaterThan(0); + expect(parseInt(failedDocs, 10)).to.be(20); }); }); @@ -371,7 +417,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const rows = await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - expect(rows.length).to.eql(3); + expect(rows.length).to.eql(2); }); it('should display Spark Plot for every row of degraded fields', async () => { diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index fb6c6ed9b519f..32a3368d435a2 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -6,8 +6,11 @@ */ import expect from '@kbn/expect'; +import merge from 'lodash/merge'; import { DatasetQualityFtrProviderContext } from './config'; import { + createFailedRecords, + customLogLevelProcessor, datasetNames, defaultNamespace, getInitialTestLogs, @@ -26,6 +29,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const to = '2024-01-01T12:00:00.000Z'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; + const failedDatasetName = 'synth.failed'; const pkg = { name: 'apache', version: '1.14.0', @@ -33,6 +37,22 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid describe('Dataset quality table', () => { before(async () => { + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor); + await synthtrace.updateIndexTemplate( + 'logs', + (template: Record): Record => { + const next: Record = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + // Install Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(pkg); // Ingest basic logs @@ -53,6 +73,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.datasetQuality.navigateTo(); }); @@ -64,7 +91,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows sort by dataset name and show namespace', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; await datasetNameCol.sort('descending'); const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql( @@ -77,6 +104,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid defaultNamespace, defaultNamespace, defaultNamespace, + defaultNamespace, productionNamespace, ]); @@ -86,13 +114,15 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows the last activity', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const activityCells = await lastActivityCol.getCellTexts(); - const lastActivityCell = activityCells[activityCells.length - 1]; - const restActivityCells = activityCells.slice(0, -1); + const lastActivityDegradedCell = activityCells[activityCells.length - 2]; + const lastActivityFailedCell = activityCells[activityCells.length - 1]; + const restActivityCells = activityCells.slice(0, -2); // The first cell of lastActivity should have data - expect(lastActivityCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityDegradedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityFailedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); // The rest of the rows must show no activity expect(restActivityCells).to.eql([ PageObjects.datasetQuality.texts.noActivityText, @@ -104,9 +134,17 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows degraded docs percentage', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const degradedDocsCol = cols['Degraded Docs (%)']; + const degradedDocsCol = cols['Degraded docs (%)']; const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts(); - expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); + expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%', '0%']); + }); + + it('shows failed docs percentage', async () => { + const cols = await PageObjects.datasetQuality.parseDatasetTable(); + + const failedDocsCol = cols['Failed docs (%)']; + const failedDocsColCellTexts = await failedDocsCol.getCellTexts(); + expect(failedDocsColCellTexts).to.eql(['0%', '0%', '0%', '0%', '20%']); }); it('shows the value in the size column', async () => { @@ -122,7 +160,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows dataset from integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); @@ -132,7 +170,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('goes to log explorer page when opened', async () => { const rowIndexToOpen = 1; const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const actionsCol = cols.Actions; const datasetName = (await datasetNameCol.getCellTexts())[rowIndexToOpen]; @@ -150,7 +188,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('hides inactive datasets', async () => { // Get number of rows with Last Activity not equal to "No activity in the selected timeframe" const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const lastActivityColCellTexts = await lastActivityCol.getCellTexts(); const activeDatasets = lastActivityColCellTexts.filter( (activity: string) => activity !== PageObjects.datasetQuality.texts.noActivityText diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts index 3c8c3e702d576..51b4163fa71fe 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts @@ -7,7 +7,13 @@ import expect from '@kbn/expect'; import { DatasetQualityFtrProviderContext } from './config'; -import { datasetNames, getInitialTestLogs, getLogsForDataset, productionNamespace } from './data'; +import { + createFailedRecords, + datasetNames, + getInitialTestLogs, + getLogsForDataset, + productionNamespace, +} from './data'; export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { const PageObjects = getPageObjects([ @@ -19,6 +25,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const synthtrace = getService('logSynthtraceEsClient'); const testSubjects = getService('testSubjects'); const to = '2024-01-01T12:00:00.000Z'; + const failedDatasetName = 'synth.failed'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; const apacheIntegrationName = 'Apache HTTP Server'; @@ -50,6 +57,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.datasetQuality.navigateTo(); }); @@ -61,7 +75,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows full dataset names when toggled', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -83,7 +97,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('searches the datasets', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -94,7 +108,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ); const colsAfterSearch = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterSearch = colsAfterSearch['Data Set Name']; + const datasetNameColAfterSearch = colsAfterSearch['Data set name']; const datasetNameColCellTextsAfterSearch = await datasetNameColAfterSearch.getCellTexts(); expect(datasetNameColCellTextsAfterSearch).to.eql([datasetNames[2]]); @@ -104,7 +118,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('filters for integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -112,7 +126,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.filterForIntegrations([apacheIntegrationName]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterFilter = colsAfterFilter['Data Set Name']; + const datasetNameColAfterFilter = colsAfterFilter['Data set name']; const datasetNameColCellTextsAfterFilter = await datasetNameColAfterFilter.getCellTexts(); expect(datasetNameColCellTextsAfterFilter).to.eql([apacheAccessDatasetHumanName]); @@ -143,7 +157,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const expectedQuality = 'Poor'; // Get default quality const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQuality = cols['Data Set Quality']; + const datasetQuality = cols['Data set quality']; const datasetQualityCellTexts = await datasetQuality.getCellTexts(); expect(datasetQualityCellTexts).to.contain(expectedQuality); @@ -151,7 +165,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.filterForQualities([expectedQuality]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQualityAfterFilter = colsAfterFilter['Data Set Quality']; + const datasetQualityAfterFilter = colsAfterFilter['Data set quality']; const datasetQualityCellTextsAfterFilter = await datasetQualityAfterFilter.getCellTexts(); expect(datasetQualityCellTextsAfterFilter).to.eql([expectedQuality]); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index cef881fe0797c..389342d0b3ecf 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -54,7 +54,7 @@ type SummaryPanelKpi = Record< >; type SummaryPanelKPI = Record< - 'docsCountTotal' | 'size' | 'services' | 'hosts' | 'degradedDocs', + 'docsCountTotal' | 'size' | 'services' | 'hosts' | 'degradedDocs' | 'failedDocs', string >; @@ -70,6 +70,7 @@ const texts = { services: 'Services', hosts: 'Hosts', degradedDocs: 'Degraded docs', + failedDocs: 'Failed docs', }; export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProviderContext) { @@ -138,6 +139,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist', datasetQualityDetailsOverviewDegradedFieldToggleSwitch: 'datasetQualityDetailsOverviewDegradedFieldToggleSwitch', + datasetQualityIssuesChartTypeButtonGroup: 'datasetQualityDetailsChartTypeButtonGroup', }; return { @@ -299,13 +301,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv await this.waitUntilTableLoaded(); const table = await this.getDatasetsTable(); return this.parseTable(table, [ - '0', - 'Data Set Name', + 'Data set name', 'Namespace', + 'Type', 'Size', - 'Data Set Quality', - 'Degraded Docs (%)', - 'Last Activity', + 'Data set quality', + 'Degraded docs (%)', + 'Failed docs (%)', + 'Last activity', 'Actions', ]); }, @@ -395,6 +398,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv { title: texts.services, key: 'services' }, { title: texts.hosts, key: 'hosts' }, { title: texts.degradedDocs, key: 'degradedDocs' }, + { title: texts.failedDocs, key: 'failedDocs' }, ].filter((item) => !excludeKeys.includes(item.key)); const kpiTexts = await Promise.all( @@ -415,6 +419,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ); }, + async selectQualityIssuesChartType(chartType: 'degradedDocs' | 'failedDocs') { + const datasetDetailsContainer: WebElementWrapper = await testSubjects.find( + testSubjectSelectors.datasetQualityIssuesChartTypeButtonGroup + ); + const refreshButton = await datasetDetailsContainer.findByTestSubject(chartType); + return refreshButton.click(); + }, + /** * Selects a breakdown field from the unified histogram breakdown selector * @param fieldText The text of the field to select. Use 'No breakdown' to clear the selection diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts index 3692a17709a2e..338177a7b6b9e 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts @@ -189,7 +189,7 @@ export function createDegradedFieldsRecord({ }); } -export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; +export const datasetNames = ['synth.1', 'synth.2', 'synth.3', 'synth.failed']; export const defaultNamespace = 'default'; export const productionNamespace = 'production'; diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts index 0d8d8e8865d52..1f0abda2242a3 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts @@ -6,7 +6,16 @@ */ import expect from '@kbn/expect'; -import { defaultNamespace } from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; +import { + createFailedRecords, + customLogLevelProcessor, + defaultNamespace, +} from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; +import merge from 'lodash/merge'; +import { + IndicesIndexTemplate, + IndicesPutIndexTemplateRequest, +} from '@elastic/elasticsearch/lib/api/types'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { datasetNames, @@ -55,6 +64,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const regularDataStreamName = `logs-${datasetNames[0]}-${defaultNamespace}`; const degradedDatasetName = datasetNames[2]; const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const failedDatasetName = datasetNames[1]; + const failedDataStreamName = `logs-${failedDatasetName}-${defaultNamespace}`; describe('Dataset quality details', function () { before(async () => { @@ -64,6 +75,31 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Install Bitbucket Integration (package which does not has Dashboards) and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(bitbucketPkg); + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor, 'logs-apache.access@custom'); + await synthtrace.createComponentTemplate('logs-apache.access@custom', undefined, { + 'index.default_pipeline': 'logs-apache.access@custom', + }); + await synthtrace.updateIndexTemplate( + 'logs-apache.access', + (template: IndicesIndexTemplate): IndicesPutIndexTemplateRequest => { + const next = { + name: 'logs-apache.access', + index_patterns: template.index_patterns, + template: { + settings: template.template?.settings, + mappings: template.template?.mappings, + aliases: template.template?.aliases, + }, + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + await synthtrace.index([ // Ingest basic logs getInitialTestLogs({ to, count: 4 }), @@ -88,6 +124,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { namespace: productionNamespace, isMalformed: true, }), + // Index failed docs for Apache integration + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: apacheAccessDatasetName, + namespace: productionNamespace, + rate: 0.5, + }), // Index logs for Bitbucket integration getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName }), ]); @@ -163,6 +207,19 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(currentUrl).to.not.contain('breakdownField'); }); }); + + it('reflects the selected quality issue chart state in url', async () => { + await PageObjects.datasetQuality.navigateToDetails({ dataStream: failedDataStreamName }); + + const charType = 'failedDocs'; + await PageObjects.datasetQuality.selectQualityIssuesChartType(charType); + + // Wait for URL to contain "qualityIssuesChart:failedDocs" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(decodeURIComponent(currentUrl)).to.contain(`qualityIssuesChart:${charType}`); + }); + }); }); // FLAKY: https://github.com/elastic/kibana/issues/194575 @@ -172,15 +229,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataStream: apacheAccessDataStreamName, }); - const { docsCountTotal, degradedDocs, services, hosts, size } = + const { docsCountTotal, degradedDocs, failedDocs, services, hosts, size } = await PageObjects.datasetQuality.parseOverviewSummaryPanelKpis(); - expect(parseInt(docsCountTotal, 10)).to.be(226); + expect(parseInt(docsCountTotal, 10)).to.be(306); expect(parseInt(degradedDocs, 10)).to.be(1); expect(parseInt(services, 10)).to.be(3); expect(parseInt(hosts, 10)).to.be(52); // metering stats API is cached for 30seconds, waiting for the exact value is not optimal in this case // rather we can just check if any value is present expect(size).to.be.ok(); + expect(parseInt(failedDocs, 10)).to.be(20); }); }); @@ -377,7 +435,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const rows = await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - expect(rows.length).to.eql(3); + expect(rows.length).to.eql(2); }); it('should display Spark Plot for every row of degraded fields', async () => { diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts index 80214767c92d2..a8e8cf9bf6b7b 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts @@ -6,7 +6,12 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; +import { + createFailedRecords, + customLogLevelProcessor, +} from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; +import merge from 'lodash/merge'; +import { IndicesIndexTemplate } from '@elastic/elasticsearch/lib/api/types'; import { datasetNames, defaultNamespace, @@ -14,6 +19,7 @@ import { getLogsForDataset, productionNamespace, } from './data'; +import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects([ @@ -28,6 +34,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const to = '2024-01-01T12:00:00.000Z'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; + const failedDatasetName = 'synth.failed'; const pkg = { name: 'apache', version: '1.14.0', @@ -35,6 +42,19 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Dataset quality table', function () { before(async () => { + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor); + await synthtrace.updateIndexTemplate('logs', (template: IndicesIndexTemplate) => { + const next = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + }); + // Install Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(pkg); // Ingest basic logs @@ -55,6 +75,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.svlCommonPage.loginAsAdmin(); await PageObjects.datasetQuality.navigateTo(); @@ -67,7 +94,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows sort by dataset name and show namespace', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; await datasetNameCol.sort('descending'); const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql( @@ -80,6 +107,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { defaultNamespace, defaultNamespace, defaultNamespace, + defaultNamespace, productionNamespace, ]); @@ -89,13 +117,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows the last activity', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const activityCells = await lastActivityCol.getCellTexts(); - const lastActivityCell = activityCells[activityCells.length - 1]; - const restActivityCells = activityCells.slice(0, -1); + const lastActivityDegradedCell = activityCells[activityCells.length - 2]; + const lastActivityFailedCell = activityCells[activityCells.length - 1]; + const restActivityCells = activityCells.slice(0, -2); // The first cell of lastActivity should have data - expect(lastActivityCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityDegradedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityFailedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); // The rest of the rows must show no activity expect(restActivityCells).to.eql([ PageObjects.datasetQuality.texts.noActivityText, @@ -107,9 +137,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows degraded docs percentage', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const degradedDocsCol = cols['Degraded Docs (%)']; + const degradedDocsCol = cols['Degraded docs (%)']; const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts(); - expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); + expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%', '0%']); + }); + + it('shows failed docs percentage', async () => { + const cols = await PageObjects.datasetQuality.parseDatasetTable(); + + const failedDocsCol = cols['Failed docs (%)']; + const failedDocsColCellTexts = await failedDocsCol.getCellTexts(); + expect(failedDocsColCellTexts).to.eql(['0%', '0%', '0%', '0%', '20%']); }); it('shows the value in the size column', async () => { @@ -125,7 +163,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows dataset from integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); @@ -135,7 +173,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('goes to log explorer page when opened', async () => { const rowIndexToOpen = 1; const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const actionsCol = cols.Actions; const datasetName = (await datasetNameCol.getCellTexts())[rowIndexToOpen]; @@ -153,7 +191,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('hides inactive datasets', async () => { // Get number of rows with Last Activity not equal to "No activity in the selected timeframe" const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const lastActivityColCellTexts = await lastActivityCol.getCellTexts(); const activeDatasets = lastActivityColCellTexts.filter( (activity) => activity !== PageObjects.datasetQuality.texts.noActivityText diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts index d57f852c0e700..972b7bb420162 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { createFailedRecords } from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { datasetNames, getInitialTestLogs, getLogsForDataset, productionNamespace } from './data'; @@ -20,6 +21,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const synthtrace = getService('svlLogsSynthtraceClient'); const testSubjects = getService('testSubjects'); const to = '2024-01-01T12:00:00.000Z'; + const failedDatasetName = 'synth.failed'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; const apacheIntegrationName = 'Apache HTTP Server'; @@ -51,6 +53,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.svlCommonPage.loginWithPrivilegedRole(); await PageObjects.datasetQuality.navigateTo(); @@ -63,7 +72,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows full dataset names when toggled', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -84,7 +93,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('searches the datasets', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -95,7 +104,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); const colsAfterSearch = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterSearch = colsAfterSearch['Data Set Name']; + const datasetNameColAfterSearch = colsAfterSearch['Data set name']; const datasetNameColCellTextsAfterSearch = await datasetNameColAfterSearch.getCellTexts(); expect(datasetNameColCellTextsAfterSearch).to.eql([datasetNames[2]]); // Reset the search field @@ -104,7 +113,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('filters for integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -112,7 +121,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.filterForIntegrations([apacheIntegrationName]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterFilter = colsAfterFilter['Data Set Name']; + const datasetNameColAfterFilter = colsAfterFilter['Data set name']; const datasetNameColCellTextsAfterFilter = await datasetNameColAfterFilter.getCellTexts(); expect(datasetNameColCellTextsAfterFilter).to.eql([apacheAccessDatasetHumanName]); // Reset the filter by selecting from the dropdown again @@ -142,7 +151,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const expectedQuality = 'Poor'; // Get default quality const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQuality = cols['Data Set Quality']; + const datasetQuality = cols['Data set quality']; const datasetQualityCellTexts = await datasetQuality.getCellTexts(); expect(datasetQualityCellTexts).to.contain(expectedQuality); @@ -150,7 +159,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.filterForQualities([expectedQuality]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQualityAfterFilter = colsAfterFilter['Data Set Quality']; + const datasetQualityAfterFilter = colsAfterFilter['Data set quality']; const datasetQualityCellTextsAfterFilter = await datasetQualityAfterFilter.getCellTexts(); expect(datasetQualityCellTextsAfterFilter).to.eql([expectedQuality]);