From 81ad3fa76e25cacf97ddac52fe5a31f56d57c2c6 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 12 Nov 2024 11:42:52 -0300 Subject: [PATCH 01/19] Fetching failed docs and showing them in the table --- .../dataset_quality/common/api_types.ts | 6 + .../dataset_quality/common/constants.ts | 2 +- .../data_streams_stats/data_stream_stat.ts | 34 ++++-- .../common/data_streams_stats/types.ts | 3 + .../common/utils/dataset_name.test.ts | 6 + .../common/utils/dataset_name.ts | 2 +- .../common/utils/quality_helpers.ts | 16 ++- .../datasets_quality_indicators.tsx | 19 +-- .../dataset_quality/table/columns.tsx | 92 ++++++++++---- .../table/failed_docs_percentage_link.tsx | 34 ++++++ .../components/quality_indicator/helpers.ts | 15 --- .../components/quality_indicator/index.ts | 1 - .../hooks/use_dataset_details_telemetry.ts | 2 +- .../hooks/use_dataset_quality_table.tsx | 1 + .../public/hooks/use_summary_panel.ts | 9 +- .../data_streams_stats_client.ts | 27 ++++ .../services/data_streams_stats/types.ts | 4 + .../src/defaults.ts | 1 + .../src/notifications.ts | 9 ++ .../src/state_machine.ts | 52 ++++++++ .../dataset_quality_controller/src/types.ts | 5 + .../public/utils/generate_datasets.test.ts | 80 ++++++++++-- .../public/utils/generate_datasets.ts | 49 ++++++-- .../routes/data_streams/get_failed_docs.ts | 115 ++++++++++++++++++ .../server/routes/data_streams/routes.ts | 59 +++++++-- .../utils/create_dataset_quality_es_client.ts | 1 + 26 files changed, 535 insertions(+), 109 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx delete mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index 51a1421aec918..a293112eed548 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -56,6 +56,12 @@ export const getDataStreamDegradedDocsResponseRt = rt.type({ export type DataStreamDegradedDocsResponse = rt.TypeOf; +export const getDataStreamFailedDocsResponseRt = rt.type({ + failedDocs: rt.array(dataStreamDocsStatRt), +}); + +export type DataStreamFailedDocsResponse = rt.TypeOf; + export const integrationDashboardRT = rt.type({ id: rt.string, title: rt.string, diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index 74809e0e19420..fca2bcc83f3ab 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -26,7 +26,7 @@ export const NONE = 'none'; export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' }; export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false }; -export const DEFAULT_DEGRADED_DOCS = { +export const DEFAULT_QUALITY_DOC_STATS = { count: 0, percentage: 0, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 094d92ff3fea6..7bf45e95c8096 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -6,12 +6,14 @@ */ import { DataStreamDocsStat } from '../api_types'; -import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants'; +import { DEFAULT_DATASET_QUALITY, DEFAULT_QUALITY_DOC_STATS } from '../constants'; import { DataStreamType, QualityIndicators } from '../types'; import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils'; import { Integration } from './integration'; import { DataStreamStatType } from './types'; +type QualityStat = Omit & { percentage: number }; + export class DataStreamStat { rawName: string; type: DataStreamType; @@ -30,6 +32,10 @@ export class DataStreamStat { percentage: number; count: number; }; + failedDocs: { + percentage: number; + count: number; + }; private constructor(dataStreamStat: DataStreamStat) { this.rawName = dataStreamStat.rawName; @@ -46,6 +52,7 @@ export class DataStreamStat { this.quality = dataStreamStat.quality; this.docsInTimeRange = dataStreamStat.docsInTimeRange; this.degradedDocs = dataStreamStat.degradedDocs; + this.failedDocs = dataStreamStat.failedDocs; } public static create(dataStreamStat: DataStreamStatType) { @@ -63,36 +70,45 @@ export class DataStreamStat { userPrivileges: dataStreamStat.userPrivileges, totalDocs: dataStreamStat.totalDocs, quality: DEFAULT_DATASET_QUALITY, - degradedDocs: DEFAULT_DEGRADED_DOCS, + degradedDocs: DEFAULT_QUALITY_DOC_STATS, + failedDocs: DEFAULT_QUALITY_DOC_STATS, }; return new DataStreamStat(dataStreamStatProps); } - public static fromDegradedDocStat({ + public static fromQualityStats({ + datasetName, degradedDocStat, + failedDocStat, datasetIntegrationMap, totalDocs, }: { - degradedDocStat: DataStreamDocsStat & { percentage: number }; + datasetName: string; + degradedDocStat: QualityStat; + failedDocStat: QualityStat; datasetIntegrationMap: Record; totalDocs: number; }) { - const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset); + const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName); const dataStreamStatProps = { - rawName: degradedDocStat.dataset, + rawName: datasetName, type, name: dataset, - title: datasetIntegrationMap[dataset]?.title || dataset, + title: datasetIntegrationMap[datasetName]?.title || dataset, namespace, - integration: datasetIntegrationMap[dataset]?.integration, - quality: mapPercentageToQuality(degradedDocStat.percentage), + integration: datasetIntegrationMap[datasetName]?.integration, + quality: mapPercentageToQuality([degradedDocStat.percentage, failedDocStat.percentage]), docsInTimeRange: totalDocs, degradedDocs: { percentage: degradedDocStat.percentage, count: degradedDocStat.count, }, + failedDocs: { + percentage: failedDocStat.percentage, + count: failedDocStat.count, + }, }; return new DataStreamStat(dataStreamStatProps); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index bc0c12d234d26..c4fee2e7f5e31 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -18,6 +18,9 @@ export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse; export type GetDataStreamsDegradedDocsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params']; export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query']; +export type GetDataStreamsFailedDocsStatsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/failed_docs`>['params']; +export type GetDataStreamsFailedDocsStatsQuery = GetDataStreamsFailedDocsStatsParams['query']; /* Types for stats based in documents inside a DataStream diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts index 8ffcbfe5657fa..ad6d2678932b8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts @@ -84,5 +84,11 @@ describe('dataset_name', () => { extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001') ).toEqual('metrics-apm.app.adservice-default'); }); + + it('returns the correct index name if backing index is a failure store index', () => { + expect( + extractIndexNameFromBackingIndex('.fs-logs-elastic_agent-default-2024.11.11-000001') + ).toEqual('logs-elastic_agent-default'); + }); }); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts index eaca58ded6404..08944df9efcdc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts @@ -40,7 +40,7 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => { }; export const extractIndexNameFromBackingIndex = (indexString: string): string => { - const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; + const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; const match = indexString.match(pattern); return match ? match[1] : indexString; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts index 62e0411e541b0..45dcbdd19ff9f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/quality_helpers.ts @@ -8,10 +8,14 @@ import { POOR_QUALITY_MINIMUM_PERCENTAGE, DEGRADED_QUALITY_MINIMUM_PERCENTAGE } from '../constants'; import { QualityIndicators } from '../types'; -export const mapPercentageToQuality = (percentage: number): QualityIndicators => { - return percentage > POOR_QUALITY_MINIMUM_PERCENTAGE - ? 'poor' - : percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE - ? 'degraded' - : 'good'; +export const mapPercentageToQuality = (percentages: number[]): QualityIndicators => { + if (percentages.some((percentage) => percentage > POOR_QUALITY_MINIMUM_PERCENTAGE)) { + return 'poor'; + } + + if (percentages.some((percentage) => percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE)) { + return 'degraded'; + } + + return 'good'; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx index b186c16c0c0f8..36cd65423d83b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx @@ -29,20 +29,11 @@ import { summaryPanelQualityText, summaryPanelQualityTooltipText, } from '../../../../common/translations'; -import { mapPercentagesToQualityCounts } from '../../quality_indicator'; export function DatasetsQualityIndicators() { const { onPageReady } = usePerformanceContext(); - const { - datasetsQuality, - isDatasetsQualityLoading, - datasetsActivity, - numberOfDatasets, - numberOfDocuments, - } = useSummaryPanelContext(); - const qualityCounts = mapPercentagesToQualityCounts(datasetsQuality.percentages); - const datasetsWithoutIgnoredField = - datasetsActivity.total > 0 ? datasetsActivity.total - datasetsQuality.percentages.length : 0; + const { datasetsQuality, isDatasetsQualityLoading, numberOfDatasets, numberOfDocuments } = + useSummaryPanelContext(); if (!isDatasetsQualityLoading && (numberOfDatasets || numberOfDocuments)) { onPageReady({ @@ -66,21 +57,21 @@ export function DatasetsQualityIndicators() { ); +const failedDocsColumnTooltip = ( + +); + const datasetQualityColumnTooltip = ( @@ -273,7 +285,7 @@ export const getDatasetQualityTableColumns = ({ ), - field: 'degradedDocs.percentage', + field: 'quality', sortable: true, render: (_, dataStreamStat: DataStreamStat) => ( @@ -304,35 +316,61 @@ export const getDatasetQualityTableColumns = ({ }, { name: ( - - {lastActivityColumnName} + + + + {`${failedDocsColumnName} `} + + + ), - field: 'lastActivity', - render: (timestamp: number) => ( - - {!isActiveDataset(timestamp) ? ( - - {inactiveDatasetActivityColumnDescription} - - - - - ) : ( - fieldFormats - .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) - .convert(timestamp) - )} - - ), - width: '300px', + field: 'failedDocs.percentage', sortable: true, + render: (_, dataStreamStat: DataStreamStat) => ( + + ), + width: '140px', }, + ...(canUserMonitorDataset && canUserMonitorAnyDataStream + ? [ + { + name: ( + + {lastActivityColumnName} + + ), + field: 'lastActivity', + render: (timestamp: number) => ( + + {!isActiveDataset(timestamp) ? ( + + {inactiveDatasetActivityColumnDescription} + + + + + ) : ( + fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) + .convert(timestamp) + )} + + ), + width: '300px', + sortable: true, + }, + ] + : []), { name: actionsColumnName, render: (dataStreamStat: DataStreamStat) => ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx new file mode 100644 index 0000000000000..32732b598066c --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiSkeletonRectangle, EuiFlexGroup } from '@elastic/eui'; +import React from 'react'; +import { QualityPercentageIndicator } from '../../quality_indicator'; +import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; +import { TimeRangeConfig } from '../../../../common/types'; + +export const FailedDocsPercentageLink = ({ + isLoading, + dataStreamStat, + timeRange, +}: { + isLoading: boolean; + dataStreamStat: DataStreamStat; + timeRange: TimeRangeConfig; +}) => { + const { + failedDocs: { percentage }, + } = dataStreamStat; + + return ( + + + + + + ); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts deleted file mode 100644 index c9588da392d96..0000000000000 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/helpers.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { countBy } from 'lodash'; -import { QualityIndicators } from '../../../common/types'; -import { mapPercentageToQuality } from '../../../common/utils'; - -export const mapPercentagesToQualityCounts = ( - percentages: number[] -): Record => - countBy(percentages.map(mapPercentageToQuality)) as Record; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts index 4f41732ca5259..f95d565f1b217 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/index.ts @@ -7,5 +7,4 @@ export * from './indicator'; export * from './percentage_indicator'; -export * from './helpers'; export * from './dataset_quality_indicator'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts index f613d3af7fdc4..e98900f77ae44 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts @@ -170,7 +170,7 @@ function getDatasetDetailsEbtProps({ const totalDocs = dataStreamDetails?.docsCount ?? 0; const degradedPercentage = totalDocs > 0 ? Number(((degradedDocs / totalDocs) * 100).toFixed(2)) : 0; - const health = mapPercentageToQuality(degradedPercentage); + const health = mapPercentageToQuality([degradedPercentage]); const { startDate: from, endDate: to } = getDateISORange(timeRange); return { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 6529ae1841ee3..30ff971043986 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -25,6 +25,7 @@ const sortingOverrides: Partial<{ }> = { ['title']: 'name', ['size']: DataStreamStat.calculateFilteredSize, + ['quality']: (item) => Math.max(item.degradedDocs.percentage, item.failedDocs.percentage), }; export const useDatasetQualityTable = () => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts index 014d9f578eb60..7a22f5fd9adb7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts @@ -7,10 +7,12 @@ import createContainer from 'constate'; import { useSelector } from '@xstate/react'; +import { countBy } from 'lodash'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { useDatasetQualityTable } from '.'; import { useDatasetQualityContext } from '../components/dataset_quality/context'; import { filterInactiveDatasets } from '../utils'; +import { QualityIndicators } from '../../common/types'; const useSummaryPanel = () => { const { service } = useDatasetQualityContext(); @@ -23,9 +25,10 @@ const useSummaryPanel = () => { Datasets Quality */ - const datasetsQuality = { - percentages: filteredItems.map((item) => item.degradedDocs.percentage), - }; + const datasetsQuality = countBy(filteredItems.map((item) => item.quality)) as Record< + QualityIndicators, + number + >; const isDatasetsQualityLoading = useSelector(service, (state) => state.matches('stats.degradedDocs.fetching') diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 8e218819315b2..b00fd5cbcac2f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -11,8 +11,10 @@ import rison from '@kbn/rison'; import { KNOWN_TYPES } from '../../../common/constants'; import { DataStreamDegradedDocsResponse, + DataStreamFailedDocsResponse, DataStreamTotalDocsResponse, getDataStreamDegradedDocsResponseRt, + getDataStreamFailedDocsResponseRt, getDataStreamsStatsResponseRt, getDataStreamTotalDocsResponseRt, getIntegrationsResponseRt, @@ -23,6 +25,7 @@ import { import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, + GetDataStreamsFailedDocsStatsQuery, GetDataStreamsStatsQuery, GetDataStreamsStatsResponse, GetDataStreamsTotalDocsQuery, @@ -108,6 +111,30 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return degradedDocs; } + public async getDataStreamsFailedStats(params: GetDataStreamsFailedDocsStatsQuery) { + const types = params.types.length === 0 ? KNOWN_TYPES : params.types; + const response = await this.http + .get('/internal/dataset_quality/data_streams/failed_docs', { + query: { + ...params, + types: rison.encodeArray(types), + }, + }) + .catch((error) => { + throw new DatasetQualityError(`Failed to fetch data streams failed stats: ${error}`, error); + }); + + const { failedDocs } = decodeOrThrow( + getDataStreamFailedDocsResponseRt, + (message: string) => + new DatasetQualityError( + `Failed to decode data streams failed docs stats response: ${message}` + ) + )(response); + + return failedDocs; + } + public async getNonAggregatableDatasets(params: GetNonAggregatableDataStreamsParams) { const types = params.types.length === 0 ? KNOWN_TYPES : params.types; const response = await this.http diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts index 240e5519cfc3d..96da4b0d0c723 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts @@ -9,6 +9,7 @@ import { HttpStart } from '@kbn/core/public'; import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, + GetDataStreamsFailedDocsStatsQuery, GetDataStreamsStatsQuery, GetDataStreamsTotalDocsQuery, GetNonAggregatableDataStreamsParams, @@ -31,6 +32,9 @@ export interface IDataStreamsStatsClient { getDataStreamsDegradedStats( params?: GetDataStreamsDegradedDocsStatsQuery ): Promise; + getDataStreamsFailedStats( + params?: GetDataStreamsFailedDocsStatsQuery + ): Promise; getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery): Promise; getIntegrations(): Promise; getNonAggregatableDatasets( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index 7c77fe9d59422..d6d266c6753b8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -38,6 +38,7 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { }, dataStreamStats: [], degradedDocStats: [], + failedDocStats: [], totalDocsStats: DEFAULT_DICTIONARY_TYPE, filters: { inactive: true, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index 0dea80104245f..f798256f65149 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -47,3 +47,12 @@ export const fetchIntegrationsFailedNotifier = (toasts: IToasts, error: Error) = text: error.message, }); }; + +export const fetchFailedStatsFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchFailedStatsFailed', { + defaultMessage: "We couldn't get your failed docs information.", + }), + text: error.message, + }); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 1217e52894ce7..9b5049c3a6c30 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -24,6 +24,7 @@ import { DEFAULT_CONTEXT } from './defaults'; import { fetchDatasetStatsFailedNotifier, fetchDegradedStatsFailedNotifier, + fetchFailedStatsFailedNotifier, fetchIntegrationsFailedNotifier, fetchTotalDocsFailedNotifier, } from './notifications'; @@ -125,6 +126,41 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }, }, + failedDocs: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadFailedDocs', + onDone: { + target: 'loaded', + actions: ['storeFailedDocStats', 'storeDatasets'], + }, + onError: [ + { + target: 'unauthorized', + cond: 'checkIfActionForbidden', + }, + { + target: 'loaded', + actions: ['notifyFetchFailedStatsFailed'], + }, + ], + }, + }, + loaded: {}, + unauthorized: { type: 'final' }, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'failedDocs.fetching', + actions: ['storeTimeRange'], + }, + REFRESH_DATA: { + target: 'failedDocs.fetching', + }, + }, + }, docsStats: { initial: 'fetching', states: { @@ -381,6 +417,9 @@ export const createPureDatasetQualityControllerStateMachine = ( storeDegradedDocStats: assign((_context, event: DoneInvokeEvent) => ({ degradedDocStats: event.data, })), + storeFailedDocStats: assign((_context, event: DoneInvokeEvent) => ({ + failedDocStats: event.data, + })), storeNonAggregatableDatasets: assign( (_context, event: DoneInvokeEvent) => ({ nonAggregatableDatasets: event.data.datasets, @@ -404,6 +443,7 @@ export const createPureDatasetQualityControllerStateMachine = ( datasets: generateDatasets( context.dataStreamStats, context.degradedDocStats, + context.failedDocStats, context.integrations, context.totalDocsStats ), @@ -447,6 +487,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchIntegrationsFailedNotifier(toasts, event.data), notifyFetchTotalDocsFailed: (_context, event: DoneInvokeEvent, meta) => fetchTotalDocsFailedNotifier(toasts, event.data, meta), + notifyFetchFailedStatsFailed: (_context, event: DoneInvokeEvent) => + fetchFailedStatsFailedNotifier(toasts, event.data), }, services: { loadDataStreamStats: (context, _event) => @@ -489,6 +531,16 @@ export const createDatasetQualityControllerStateMachine = ({ end, }); }, + loadFailedDocs: (context) => { + const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); + + return dataStreamStatsClient.getDataStreamsFailedStats({ + types: context.filters.types as DataStreamType[], + datasetQuery: context.filters.query, + start, + end, + }); + }, loadNonAggregatableDatasets: (context) => { const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index de7fdbf9fbd77..4f3c6716fcf24 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -60,6 +60,10 @@ export interface WithDegradedDocs { degradedDocStats: DataStreamDocsStat[]; } +export interface WithFailedDocs { + failedDocStats: DataStreamDocsStat[]; +} + export interface WithNonAggregatableDatasets { nonAggregatableDatasets: string[]; } @@ -76,6 +80,7 @@ export type DefaultDatasetQualityControllerState = WithTableOptions & WithDataStreamStats & WithTotalDocs & WithDegradedDocs & + WithFailedDocs & WithDatasets & WithFilters & WithNonAggregatableDatasets & diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index b75c74c2fd728..64e83db290674 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -73,18 +73,27 @@ describe('generateDatasets', () => { }; const degradedDocs = [ - { - dataset: 'logs-system.application-default', - count: 0, - }, { dataset: 'logs-synth-default', count: 6, }, ]; - it('merges integrations information with dataStreamStats and degradedDocs', () => { - const datasets = generateDatasets(dataStreamStats, degradedDocs, integrations, totalDocs); + const failedDocs = [ + { + dataset: 'logs-system.application-default', + count: 2, + }, + ]; + + it('merges integrations information with dataStreamStats, degradedDocs and failedDocs', () => { + const datasets = generateDatasets( + dataStreamStats, + degradedDocs, + failedDocs, + integrations, + totalDocs + ); expect(datasets).toEqual([ { @@ -102,11 +111,15 @@ describe('generateDatasets', () => { canMonitor: true, }, docsInTimeRange: 100, - quality: 'good', + quality: 'degraded', degradedDocs: { percentage: 0, count: 0, }, + failedDocs: { + percentage: 2, + count: 2, + }, }, { name: 'synth', @@ -128,6 +141,10 @@ describe('generateDatasets', () => { count: 6, percentage: 6, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); @@ -136,6 +153,7 @@ describe('generateDatasets', () => { const datasets = generateDatasets( dataStreamStats, degradedDocs, + failedDocs, integrations, DEFAULT_DICTIONARY_TYPE ); @@ -161,6 +179,10 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 2, + }, }, { name: 'synth', @@ -182,12 +204,16 @@ describe('generateDatasets', () => { count: 6, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); it('merges integrations information with degradedDocs', () => { - const datasets = generateDatasets([], degradedDocs, integrations, totalDocs); + const datasets = generateDatasets([], degradedDocs, [], integrations, totalDocs); expect(datasets).toEqual([ { @@ -208,6 +234,10 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'synth', @@ -227,12 +257,16 @@ describe('generateDatasets', () => { count: 6, percentage: 6, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); it('merges integrations information with degradedDocs and totalDocs', () => { - const datasets = generateDatasets([], degradedDocs, integrations, { + const datasets = generateDatasets([], degradedDocs, [], integrations, { ...totalDocs, logs: [...totalDocs.logs, { dataset: 'logs-another-default', count: 100 }], }); @@ -256,6 +290,10 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'synth', @@ -275,6 +313,10 @@ describe('generateDatasets', () => { count: 6, percentage: 6, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'another', @@ -294,12 +336,16 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); it('merges integrations information with dataStreamStats', () => { - const datasets = generateDatasets(dataStreamStats, [], integrations, totalDocs); + const datasets = generateDatasets(dataStreamStats, [], [], integrations, totalDocs); expect(datasets).toEqual([ { @@ -322,6 +368,10 @@ describe('generateDatasets', () => { count: 0, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, { name: 'synth', @@ -343,6 +393,10 @@ describe('generateDatasets', () => { count: 0, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); @@ -360,7 +414,7 @@ describe('generateDatasets', () => { }, }; - const datasets = generateDatasets([nonDefaultDataset], [], integrations, totalDocs); + const datasets = generateDatasets([nonDefaultDataset], [], [], integrations, totalDocs); expect(datasets).toEqual([ { @@ -383,6 +437,10 @@ describe('generateDatasets', () => { count: 0, percentage: 0, }, + failedDocs: { + percentage: 0, + count: 0, + }, }, ]); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 8e9f2f3db7083..2efeb34d42da7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { DEFAULT_DEGRADED_DOCS } from '../../common/constants'; +import { DEFAULT_QUALITY_DOC_STATS } from '../../common/constants'; import { DataStreamDocsStat } from '../../common/api_types'; import { DataStreamStatType } from '../../common/data_streams_stats/types'; import { mapPercentageToQuality } from '../../common/utils'; @@ -13,9 +13,11 @@ import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { DictionaryType } from '../state_machines/dataset_quality_controller/src/types'; import { flattenStats } from './flatten_stats'; + export function generateDatasets( dataStreamStats: DataStreamStatType[] = [], degradedDocStats: DataStreamDocsStat[] = [], + failedDocStats: DataStreamDocsStat[] = [], integrations: Integration[], totalDocsStats: DictionaryType ): DataStreamStat[] { @@ -71,12 +73,40 @@ export function generateDatasets( {} ); + const failedMap: Record< + DataStreamDocsStat['dataset'], + { + percentage: number; + count: DataStreamDocsStat['count']; + } + > = failedDocStats.reduce( + (failedMapAcc, { dataset, count }) => + Object.assign(failedMapAcc, { + [dataset]: { + count, + percentage: DataStreamStat.calculatePercentage({ + totalDocs: totalDocsMap[dataset] ? totalDocsMap[dataset] + count : 0, + count, + }), + }, + }), + {} + ); + if (!dataStreamStats.length) { - // We want to pick up all datasets even when they don't have degraded docs - const dataStreams = [...new Set([...Object.keys(totalDocsMap), ...Object.keys(degradedMap)])]; + // We want to pick up all datasets even when they don't have degraded docs or failed docs + const dataStreams = [ + ...new Set([ + ...Object.keys(totalDocsMap), + ...Object.keys(degradedMap), + ...Object.keys(failedMap), + ]), + ]; return dataStreams.map((dataset) => - DataStreamStat.fromDegradedDocStat({ - degradedDocStat: { dataset, ...(degradedMap[dataset] || DEFAULT_DEGRADED_DOCS) }, + DataStreamStat.fromQualityStats({ + datasetName: dataset, + degradedDocStat: degradedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, + failedDocStat: failedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, datasetIntegrationMap, totalDocs: totalDocsMap[dataset] ?? 0, }) @@ -85,6 +115,10 @@ export function generateDatasets( return dataStreamStats?.map((dataStream) => { const dataset = DataStreamStat.create(dataStream); + const qualityStats = [ + (degradedMap[dataset.rawName] || dataset.degradedDocs).percentage, + (failedMap[dataset.rawName] || dataset.failedDocs).percentage, + ]; return { ...dataset, @@ -93,10 +127,9 @@ export function generateDatasets( datasetIntegrationMap[dataset.name]?.integration ?? integrationsMap[dataStream.integration ?? ''], degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs, + failedDocs: failedMap[dataset.rawName] || dataset.failedDocs, docsInTimeRange: totalDocsMap[dataset.rawName] ?? 0, - quality: mapPercentageToQuality( - (degradedMap[dataset.rawName] || dataset.degradedDocs).percentage - ), + quality: mapPercentageToQuality(qualityStats), }; }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts new file mode 100644 index 0000000000000..ca93bd8c01d6b --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { rangeQuery } from '@kbn/observability-plugin/server'; +import { extractIndexNameFromBackingIndex, streamPartsToIndexPattern } from '../../../common/utils'; +import { DataStreamType } from '../../../common/types'; +import { DataStreamDocsStat } from '../../../common/api_types'; +import { createDatasetQualityESClient } from '../../utils'; +import { DatasetQualityESClient } from '../../utils/create_dataset_quality_es_client'; + +const SIZE_LIMIT = 10000; + +async function getPaginatedResults(options: { + datasetQualityESClient: DatasetQualityESClient; + index: string; + start: number; + end: number; + after?: { dataset: string }; + prevResults?: Record; +}) { + const { datasetQualityESClient, index, start, end, after, prevResults = {} } = options; + + const bool = { + filter: [...rangeQuery(start, end)], + }; + + const response = await datasetQualityESClient.search({ + index, + size: 0, + query: { + bool, + }, + aggs: { + datasets: { + composite: { + ...(after ? { after } : {}), + size: SIZE_LIMIT, + sources: [{ dataset: { terms: { field: '_index' } } }], + }, + }, + }, + failure_store: 'only', + }); + + const currResults = (response.aggregations?.datasets.buckets ?? []).reduce((acc, curr) => { + const datasetName = extractIndexNameFromBackingIndex(curr.key.dataset as string); + + return { + ...acc, + [datasetName]: (acc[datasetName] ?? 0) + curr.doc_count, + }; + }, {} as Record); + + const results = { + ...prevResults, + ...currResults, + }; + + if ( + response.aggregations?.datasets.after_key && + response.aggregations?.datasets.buckets.length === SIZE_LIMIT + ) { + return getPaginatedResults({ + datasetQualityESClient, + index, + start, + end, + after: + (response.aggregations?.datasets.after_key as { + dataset: string; + }) || after, + prevResults: results, + }); + } + + return results; +} + +export async function getFailedDocsPaginated(options: { + esClient: ElasticsearchClient; + types: DataStreamType[]; + datasetQuery?: string; + start: number; + end: number; +}): Promise { + const { esClient, types, datasetQuery, start, end } = options; + + const datasetNames = datasetQuery + ? [datasetQuery] + : types.map((type) => + streamPartsToIndexPattern({ + typePattern: type, + datasetPattern: '*-*', + }) + ); + + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const datasets = await getPaginatedResults({ + datasetQualityESClient, + index: datasetNames.join(','), + start, + end, + }); + + return Object.entries(datasets).map(([dataset, count]) => ({ + dataset, + count, + })); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 3a60f0b9a8ef3..aad3e85e448a4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -8,32 +8,33 @@ import * as t from 'io-ts'; import { DataStreamDetails, + DataStreamDocsStat, + DataStreamRolloverResponse, DataStreamSettings, DataStreamStat, - NonAggregatableDatasets, - DegradedFieldResponse, DatasetUserPrivileges, - DegradedFieldValues, DegradedFieldAnalysis, - DataStreamDocsStat, + DegradedFieldResponse, + DegradedFieldValues, + NonAggregatableDatasets, UpdateFieldLimitResponse, - DataStreamRolloverResponse, } from '../../../common/api_types'; +import { datasetQualityPrivileges } from '../../services'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; +import { createDatasetQualityESClient } from '../../utils'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; -import { datasetQualityPrivileges } from '../../services'; import { getDataStreamDetails, getDataStreamSettings } from './get_data_stream_details'; import { getDataStreams } from './get_data_streams'; +import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; import { getDataStreamsStats } from './get_data_streams_stats'; +import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; import { getDegradedDocsPaginated } from './get_degraded_docs'; -import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; -import { getDegradedFields } from './get_degraded_fields'; -import { getDegradedFieldValues } from './get_degraded_field_values'; import { analyzeDegradedField } from './get_degraded_field_analysis'; -import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; -import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; +import { getDegradedFieldValues } from './get_degraded_field_values'; +import { getDegradedFields } from './get_degraded_fields'; +import { getFailedDocsPaginated } from './get_failed_docs'; +import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; import { updateFieldLimit } from './update_field_limit'; -import { createDatasetQualityESClient } from '../../utils'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -126,6 +127,39 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ }, }); +const failedDocsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/failed_docs', + params: t.type({ + query: t.intersection([ + rangeRt, + t.type({ types: typesRt }), + t.partial({ + datasetQuery: t.string, + }), + ]), + }), + options: { + tags: [], + }, + async handler(resources): Promise<{ + failedDocs: DataStreamDocsStat[]; + }> { + const { context, params } = resources; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const failedDocs = await getFailedDocsPaginated({ + esClient, + ...params.query, + }); + + return { + failedDocs, + }; + }, +}); + const totalDocsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/total_docs', params: t.type({ @@ -411,6 +445,7 @@ const rolloverDataStream = createDatasetQualityServerRoute({ export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, + ...failedDocsRoute, ...totalDocsRoute, ...nonAggregatableDatasetsRoute, ...nonAggregatableDatasetRoute, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 8a78b4163da95..24a96ab9b2911 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -22,6 +22,7 @@ import { type DatasetQualityESSearchParams = ESSearchRequest & { size: number; + failure_store?: 'only'; }; export type DatasetQualityESClient = ReturnType; From 2ad742f790c8477cc930192682065402bf7d01ed Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 12 Nov 2024 15:54:22 -0300 Subject: [PATCH 02/19] fixing tests --- .../common/data_streams_stats/data_stream_stat.ts | 4 ++-- .../dataset_quality/public/utils/generate_datasets.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 7bf45e95c8096..40fa5d682b8bd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -96,9 +96,9 @@ export class DataStreamStat { rawName: datasetName, type, name: dataset, - title: datasetIntegrationMap[datasetName]?.title || dataset, + title: datasetIntegrationMap[dataset]?.title || dataset, namespace, - integration: datasetIntegrationMap[datasetName]?.integration, + integration: datasetIntegrationMap[dataset]?.integration, quality: mapPercentageToQuality([degradedDocStat.percentage, failedDocStat.percentage]), docsInTimeRange: totalDocs, degradedDocs: { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index 64e83db290674..c910c0b15fc96 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -117,7 +117,7 @@ describe('generateDatasets', () => { count: 0, }, failedDocs: { - percentage: 2, + percentage: 1.9607843137254901, count: 2, }, }, From 4fe638e988941cde991e44590a387cebd5b93a74 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 12 Nov 2024 16:44:03 -0300 Subject: [PATCH 03/19] Addressing copies across table and page --- .../dataset_quality/common/translations.ts | 4 +- .../dataset_quality/table/columns.tsx | 12 +++--- .../table/degraded_docs_percentage_link.tsx | 17 +++++++- .../table/failed_docs_percentage_link.tsx | 17 +++++++- .../percentage_indicator.tsx | 39 ++++++++++--------- 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 1026dd8ea58d3..53f52975d8d98 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -151,14 +151,14 @@ export const summaryPanelLast24hText = i18n.translate( export const summaryPanelQualityText = i18n.translate( 'xpack.datasetQuality.summaryPanelQualityText', { - defaultMessage: 'Data Sets Quality', + defaultMessage: 'Data Set Quality', } ); export const summaryPanelQualityTooltipText = i18n.translate( 'xpack.datasetQuality.summaryPanelQualityTooltipText', { - defaultMessage: 'Quality is based on the percentage of degraded docs in a data set.', + defaultMessage: 'Quality is based on the percentage of degraded and failed docs in a data set.', } ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 8e39352312625..17a3feee71701 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -40,7 +40,7 @@ import { DatasetQualityDetailsLink } from './dataset_quality_details_link'; import { FailedDocsPercentageLink } from './failed_docs_percentage_link'; const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', { - defaultMessage: 'Data Set Name', + defaultMessage: 'Data set name', }); const namespaceColumnName = i18n.translate('xpack.datasetQuality.namespaceColumnName', { @@ -56,19 +56,19 @@ const sizeColumnName = i18n.translate('xpack.datasetQuality.sizeColumnName', { }); const degradedDocsColumnName = i18n.translate('xpack.datasetQuality.degradedDocsColumnName', { - defaultMessage: 'Degraded Docs (%)', + defaultMessage: 'Degraded docs (%)', }); const failedDocsColumnName = i18n.translate('xpack.datasetQuality.failedDocsColumnName', { - defaultMessage: 'Failed Docs (%)', + defaultMessage: 'Failed docs (%)', }); const datasetQualityColumnName = i18n.translate('xpack.datasetQuality.datasetQualityColumnName', { - defaultMessage: 'Data Set Quality', + defaultMessage: 'Data set quality', }); const lastActivityColumnName = i18n.translate('xpack.datasetQuality.lastActivityColumnName', { - defaultMessage: 'Last Activity', + defaultMessage: 'Last activity', }); const actionsColumnName = i18n.translate('xpack.datasetQuality.actionsColumnName', { @@ -120,7 +120,7 @@ const degradedDocsColumnTooltip = ( const failedDocsColumnTooltip = ( ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx index 9d32c84891a34..6097736a857de 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx @@ -7,6 +7,7 @@ import { EuiSkeletonRectangle, EuiFlexGroup, EuiLink } from '@elastic/eui'; import React from 'react'; +import { i18n } from '@kbn/i18n'; import { _IGNORED } from '../../../../common/es_fields'; import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks'; import { QualityPercentageIndicator } from '../../quality_indicator'; @@ -38,6 +39,14 @@ export const DegradedDocsPercentageLink = ({ timeRangeConfig: timeRange, }); + const tooltip = (degradedDocsCount: number) => + i18n.translate('xpack.datasetQuality.fewDegradedDocsTooltip', { + defaultMessage: '{degradedDocsCount} degraded docs in this data set.', + values: { + degradedDocsCount, + }, + }); + return ( @@ -46,10 +55,14 @@ export const DegradedDocsPercentageLink = ({ data-test-subj="datasetQualityDegradedDocsPercentageLink" {...redirectLinkProps.linkProps} > - + ) : ( - + )} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx index 32732b598066c..4f7db1dbbeb9d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx @@ -7,6 +7,7 @@ import { EuiSkeletonRectangle, EuiFlexGroup } from '@elastic/eui'; import React from 'react'; +import { i18n } from '@kbn/i18n'; import { QualityPercentageIndicator } from '../../quality_indicator'; import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; import { TimeRangeConfig } from '../../../../common/types'; @@ -21,13 +22,25 @@ export const FailedDocsPercentageLink = ({ timeRange: TimeRangeConfig; }) => { const { - failedDocs: { percentage }, + failedDocs: { percentage, count }, } = dataStreamStat; + const tooltip = (failedDocsCount: number) => + i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', { + defaultMessage: '{failedDocsCount} failed docs in this data set.', + values: { + failedDocsCount, + }, + }); + return ( - + ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx index c10a96672598a..8f607b65b5e9d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx @@ -10,43 +10,46 @@ import { i18n } from '@kbn/i18n'; import { FormattedNumber } from '@kbn/i18n-react'; import React from 'react'; -const FEW_DEGRADED_DOCS_THRESHOLD = 0.0005; +const FEW_DOCS_THRESHOLD = 0.0005; export function QualityPercentageIndicator({ percentage, - degradedDocsCount, + docsCount, + tooltipContent, }: { percentage: number; - degradedDocsCount?: number; + docsCount?: number; + tooltipContent: (numberOfDocuments: number) => string; }) { - const isFewDegradedDocsAvailable = percentage && percentage < FEW_DEGRADED_DOCS_THRESHOLD; + const isFewDocsAvailable = percentage && percentage < FEW_DOCS_THRESHOLD; - return isFewDegradedDocsAvailable ? ( - + return isFewDocsAvailable ? ( + ) : ( - + ); } -const DatasetWithFewDegradedDocs = ({ degradedDocsCount }: { degradedDocsCount?: number }) => { +const DatasetWithFewDocs = ({ + docsCount, + tooltipContent, +}: { + docsCount: number; + tooltipContent: (numberOfDocuments: number) => string; +}) => { return ( - ~0%{' '} - + {i18n.translate('xpack.datasetQuality.datasetWithFewDocs.TextLabel', { + defaultMessage: '~0%', + })}{' '} + ); }; -const DatasetWithManyDegradedDocs = ({ percentage }: { percentage: number }) => { +const DatasetWithManyDocs = ({ percentage }: { percentage: number }) => { return ( % From e11fabf57ecd3409fab9ea520c9569f7eedb9a26 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Wed, 13 Nov 2024 07:31:47 -0300 Subject: [PATCH 04/19] Column names changed in tests --- .../apps/dataset_quality/dataset_quality_table.ts | 12 ++++++------ .../dataset_quality/dataset_quality_table.ts | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index fb6c6ed9b519f..233a7b5e6b3ba 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -64,7 +64,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows sort by dataset name and show namespace', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; await datasetNameCol.sort('descending'); const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql( @@ -86,7 +86,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows the last activity', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const activityCells = await lastActivityCol.getCellTexts(); const lastActivityCell = activityCells[activityCells.length - 1]; const restActivityCells = activityCells.slice(0, -1); @@ -104,7 +104,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows degraded docs percentage', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const degradedDocsCol = cols['Degraded Docs (%)']; + const degradedDocsCol = cols['Degraded docs (%)']; const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts(); expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); }); @@ -122,7 +122,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows dataset from integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); @@ -132,7 +132,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('goes to log explorer page when opened', async () => { const rowIndexToOpen = 1; const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const actionsCol = cols.Actions; const datasetName = (await datasetNameCol.getCellTexts())[rowIndexToOpen]; @@ -150,7 +150,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('hides inactive datasets', async () => { // Get number of rows with Last Activity not equal to "No activity in the selected timeframe" const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const lastActivityColCellTexts = await lastActivityCol.getCellTexts(); const activeDatasets = lastActivityColCellTexts.filter( (activity: string) => activity !== PageObjects.datasetQuality.texts.noActivityText diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts index 80214767c92d2..c42b50116ea95 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts @@ -67,7 +67,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows sort by dataset name and show namespace', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; await datasetNameCol.sort('descending'); const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql( @@ -89,7 +89,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows the last activity', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const activityCells = await lastActivityCol.getCellTexts(); const lastActivityCell = activityCells[activityCells.length - 1]; const restActivityCells = activityCells.slice(0, -1); @@ -107,7 +107,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows degraded docs percentage', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const degradedDocsCol = cols['Degraded Docs (%)']; + const degradedDocsCol = cols['Degraded docs (%)']; const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts(); expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); }); @@ -125,7 +125,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows dataset from integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); @@ -135,7 +135,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('goes to log explorer page when opened', async () => { const rowIndexToOpen = 1; const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const actionsCol = cols.Actions; const datasetName = (await datasetNameCol.getCellTexts())[rowIndexToOpen]; @@ -153,7 +153,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('hides inactive datasets', async () => { // Get number of rows with Last Activity not equal to "No activity in the selected timeframe" const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const lastActivityCol = cols['Last Activity']; + const lastActivityCol = cols['Last activity']; const lastActivityColCellTexts = await lastActivityCol.getCellTexts(); const activeDatasets = lastActivityColCellTexts.filter( (activity) => activity !== PageObjects.datasetQuality.texts.noActivityText From 41d249a24ccb3521ccc4a0aa356a1a39ff3f6bd0 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Thu, 14 Nov 2024 10:13:25 -0300 Subject: [PATCH 05/19] Adding failed info to dataset details --- .../dataset_quality/common/api_types.ts | 1 + .../dataset_quality/common/translations.ts | 31 +- .../document_trends/degraded_docs/index.tsx | 103 ++++--- .../failed_docs/lens_attributes.ts | 276 ++++++++++++++++++ .../overview/summary/index.tsx | 33 ++- .../overview/summary/panel.tsx | 32 +- .../public/hooks/use_failed_docs_chart.ts | 227 ++++++++++++++ .../hooks/use_overview_summary_panel.ts | 14 +- .../get_data_stream_details/index.ts | 10 + .../dataset_quality/failed_docs.ts | 167 +++++++++++ .../observability/dataset_quality/index.ts | 1 + 11 files changed, 846 insertions(+), 49 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_failed_docs_chart.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index a293112eed548..1ddd0bdbcc9c0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -188,6 +188,7 @@ export type DataStreamSettings = rt.TypeOf; export const dataStreamDetailsRt = rt.partial({ lastActivity: rt.number, degradedDocsCount: rt.number, + failedDocsCount: rt.number, docsCount: rt.number, sizeBytes: rt.number, services: rt.record(rt.string, rt.array(rt.string)), diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 53f52975d8d98..5709687726267 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -93,12 +93,9 @@ export const flyoutSummaryText = i18n.translate('xpack.datasetQuality.flyoutSumm defaultMessage: 'Summary', }); -export const overviewDegradedDocsText = i18n.translate( - 'xpack.datasetQuality.flyout.degradedDocsTitle', - { - defaultMessage: 'Degraded docs', - } -); +export const overviewTrendsDocsText = i18n.translate('xpack.datasetQuality.flyout.trendDocsTitle', { + defaultMessage: 'Document trends', +}); export const flyoutDegradedDocsTrendText = i18n.translate( 'xpack.datasetQuality.flyoutDegradedDocsViz', @@ -107,6 +104,13 @@ export const flyoutDegradedDocsTrendText = i18n.translate( } ); +export const flyoutFailedDocsTrendText = i18n.translate( + 'xpack.datasetQuality.flyoutFailedDocsViz', + { + defaultMessage: 'Failed documents trend', + } +); + export const flyoutDegradedDocsPercentageText = i18n.translate( 'xpack.datasetQuality.flyoutDegradedDocsPercentage', { @@ -115,6 +119,14 @@ export const flyoutDegradedDocsPercentageText = i18n.translate( } ); +export const flyoutFailedDocsPercentageText = i18n.translate( + 'xpack.datasetQuality.flyoutFailedDocsPercentage', + { + defaultMessage: 'Failed docs %', + description: 'Tooltip label for the percentage of failed documents chart.', + } +); + export const flyoutDocsCountTotalText = i18n.translate( 'xpack.datasetQuality.flyoutDocsCountTotal', { @@ -319,6 +331,13 @@ export const overviewPanelDatasetQualityIndicatorDegradedDocs = i18n.translate( } ); +export const overviewPanelDatasetQualityIndicatorFailedDocs = i18n.translate( + 'xpack.datasetQuality.details.overviewPanel.datasetQuality.failedDocs', + { + defaultMessage: 'Failed docs', + } +); + export const overviewDegradedFieldsTableLoadingText = i18n.translate( 'xpack.datasetQuality.details.degradedFieldsTableLoadingText', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx index 05de567a6dab7..347c970577446 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx @@ -9,8 +9,8 @@ import React, { useCallback, useEffect, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, + EuiButtonGroup, EuiButtonIcon, - EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, @@ -25,12 +25,13 @@ import { import type { DataViewField } from '@kbn/data-views-plugin/common'; import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; +import { useFailedDocsChart } from '../../../../../hooks/use_failed_docs_chart'; import { discoverAriaText, logsExplorerAriaText, openInDiscoverText, openInLogsExplorerText, - overviewDegradedDocsText, + overviewTrendsDocsText, } from '../../../../../../common/translations'; import { DegradedDocsChart } from './degraded_docs_chart'; import { @@ -42,53 +43,62 @@ import { import { _IGNORED } from '../../../../../../common/es_fields'; import { NavigationSource } from '../../../../../services/telemetry'; -const degradedDocsTooltip = ( +const trendDocsTooltip = ( - _ignored - - ), - }} + id="xpack.datasetQuality.details.trendDocsTooltip" + defaultMessage="The percentage of ignored fields or failed docs over the selected timeframe." /> ); +const DEGRADED_DOCS_KUERY = `${_IGNORED}: *`; + // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: number }) { const { timeRange, updateTimeRange, datasetDetails } = useDatasetQualityDetailsState(); - const { dataView, breakdown, ...chartProps } = useDegradedDocsChart(); + const { + dataView: degradedDataView, + breakdown: degradedBreakdown, + ...degradeChartProps + } = useDegradedDocsChart(); + const { + dataView: failedDataView, + breakdown: failedBreakDown, + ...failedChartProps + } = useFailedDocsChart(); const accordionId = useGeneratedHtmlId({ - prefix: overviewDegradedDocsText, + prefix: overviewTrendsDocsText, }); const [breakdownDataViewField, setBreakdownDataViewField] = useState( undefined ); + const [selectedChart, setSelectedChart] = useState('degradedDocs'); + const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ - query: { language: 'kuery', query: `${_IGNORED}: *` }, + query: { language: 'kuery', query: DEGRADED_DOCS_KUERY }, navigationSource: NavigationSource.Trend, }); const degradedDocLinkLogsExplorer = useRedirectLink({ dataStreamStat: datasetDetails, timeRangeConfig: timeRange, - query: { language: 'kuery', query: `${_IGNORED}: *` }, + query: { + language: 'kuery', + query: selectedChart === 'degradedDocs' ? DEGRADED_DOCS_KUERY : '', + }, sendTelemetry, }); useEffect(() => { - if (breakdown.dataViewField && breakdown.fieldSupportsBreakdown) { - setBreakdownDataViewField(breakdown.dataViewField); + if (degradedBreakdown.dataViewField && degradedBreakdown.fieldSupportsBreakdown) { + setBreakdownDataViewField(degradedBreakdown.dataViewField); } else { setBreakdownDataViewField(undefined); } - }, [setBreakdownDataViewField, breakdown]); + }, [setBreakdownDataViewField, degradedBreakdown]); const onTimeRangeChange = useCallback( ({ start, end }: Pick) => { @@ -107,9 +117,9 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe `} > -
{overviewDegradedDocsText}
+
{overviewTrendsDocsText}
 - + @@ -124,14 +134,30 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe initialIsOpen={true} data-test-subj="datasetQualityDetailsOverviewDocumentTrends" > + - - + - + + {selectedChart === 'degradedDocs' ? ( + + + + ) : ( + <> + )} - - - + {selectedChart === 'degradedDocs' ? ( + + ) : ( + + )} ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts new file mode 100644 index 0000000000000..1479a98b799e3 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts @@ -0,0 +1,276 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public'; +import { v4 as uuidv4 } from 'uuid'; + +import { + flyoutFailedDocsTrendText, + flyoutFailedDocsPercentageText, +} from '../../../../../../common/translations'; + +enum DatasetQualityLensColumn { + Date = 'date_column', + FailedDocs = 'failed_docs_column', + CountFailed = 'count_failed', + CountTotal = 'count_total', + Math = 'math_column', + Breakdown = 'breakdown_column', +} + +const MAX_BREAKDOWN_SERIES = 5; + +interface GetLensAttributesParams { + color: string; + dataStream: string; + datasetTitle: string; + breakdownFieldName?: string; +} + +export function getLensAttributes({ + color, + dataStream, + datasetTitle, + breakdownFieldName, +}: GetLensAttributesParams) { + const dataViewId = uuidv4(); + + const columnOrder = [ + DatasetQualityLensColumn.Date, + DatasetQualityLensColumn.CountFailed, + DatasetQualityLensColumn.CountTotal, + DatasetQualityLensColumn.Math, + DatasetQualityLensColumn.FailedDocs, + ]; + + if (breakdownFieldName) { + columnOrder.unshift(DatasetQualityLensColumn.Breakdown); + } + + const columns = getChartColumns(breakdownFieldName); + return { + visualizationType: 'lnsXY', + title: flyoutFailedDocsTrendText, + references: [], + state: { + ...getAdHocDataViewState(dataViewId, dataStream, datasetTitle), + datasourceStates: { + formBased: { + layers: { + layer1: { + columnOrder, + columns, + indexPatternId: dataViewId, + }, + }, + }, + }, + filters: [], + query: { + language: 'kuery', + query: '', + }, + visualization: { + axisTitlesVisibilitySettings: { + x: false, + yLeft: false, + yRight: false, + }, + fittingFunction: 'None', + gridlinesVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + layers: [ + { + accessors: [DatasetQualityLensColumn.FailedDocs], + layerId: 'layer1', + layerType: 'data', + seriesType: 'line', + xAccessor: DatasetQualityLensColumn.Date, + ...(breakdownFieldName + ? { splitAccessor: DatasetQualityLensColumn.Breakdown } + : { + yConfig: [ + { + forAccessor: DatasetQualityLensColumn.FailedDocs, + color, + }, + ], + }), + }, + ], + legend: { + isVisible: true, + position: 'right', + legendSize: 'large', + shouldTruncate: true, + }, + preferredSeriesType: 'line', + tickLabelsVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + valueLabels: 'hide', + yLeftExtent: { + mode: 'custom', + lowerBound: 0, + upperBound: undefined, + }, + }, + }, + } as TypedLensByValueInput['attributes']; +} + +function getAdHocDataViewState(id: string, dataStream: string, title: string) { + return { + internalReferences: [ + { + id, + name: 'indexpattern-datasource-current-indexpattern', + type: 'index-pattern', + }, + { + id, + name: 'indexpattern-datasource-layer-layer1', + type: 'index-pattern', + }, + ], + adHocDataViews: { + [id]: { + id, + title: dataStream, + timeFieldName: '@timestamp', + sourceFilters: [], + fieldFormats: {}, + runtimeFieldMap: {}, + fieldAttrs: {}, + allowNoIndex: false, + name: title, + }, + }, + }; +} + +function getChartColumns(breakdownField?: string): Record { + return { + [DatasetQualityLensColumn.Date]: { + dataType: 'date', + isBucketed: true, + label: '@timestamp', + operationType: 'date_histogram', + params: { + interval: 'auto', + }, + scale: 'interval', + sourceField: '@timestamp', + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.CountFailed]: { + label: '', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + filter: { + query: '', + language: 'kuery', + }, + params: { + emptyAsNull: false, + }, + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.CountTotal]: { + label: '', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + params: { + emptyAsNull: false, + }, + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.Math]: { + label: '', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'divide', + args: ['count_failed', 'count_total'], + location: { + min: 0, + max: 34, + }, + text: "count(kql='') / count()", + }, + }, + references: ['count_failed', 'count_total'], + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.FailedDocs]: { + label: flyoutFailedDocsPercentageText, + customLabel: true, + operationType: 'formula', + dataType: 'number', + references: [DatasetQualityLensColumn.Math], + isBucketed: false, + params: { + formula: "count(kql='') / count()", + format: { + id: 'percent', + params: { + decimals: 3, + }, + }, + isFormulaBroken: false, + }, + } as GenericIndexPatternColumn, + ...(breakdownField + ? { + [DatasetQualityLensColumn.Breakdown]: { + dataType: 'number', + isBucketed: true, + label: getFlyoutDegradedDocsTopNText(MAX_BREAKDOWN_SERIES, breakdownField), + operationType: 'terms', + scale: 'ordinal', + sourceField: breakdownField, + params: { + size: MAX_BREAKDOWN_SERIES, + orderBy: { + type: 'alphabetical', + fallback: true, + }, + orderDirection: 'desc', + otherBucket: true, + missingBucket: true, + parentFormat: { + id: 'terms', + }, + }, + } as GenericIndexPatternColumn, + } + : {}), + }; +} + +const getFlyoutDegradedDocsTopNText = (count: number, fieldName: string) => + i18n.translate('xpack.datasetQuality.details.degradedDocsTopNValues', { + defaultMessage: 'Top {count} values of {fieldName}', + values: { count, fieldName }, + description: + 'Tooltip label for the top N values of a field in the degraded documents trend chart.', + }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 897efb821ff64..13344ceb91767 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -6,10 +6,12 @@ */ import React from 'react'; -import { EuiFlexGroup } from '@elastic/eui'; +import { EuiCode, EuiFlexGroup } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { Panel, PanelIndicator } from './panel'; import { overviewPanelDatasetQualityIndicatorDegradedDocs, + overviewPanelDatasetQualityIndicatorFailedDocs, overviewPanelDocumentsIndicatorSize, overviewPanelDocumentsIndicatorTotalCount, overviewPanelResourcesIndicatorServices, @@ -21,6 +23,27 @@ import { import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel'; import { DatasetQualityIndicator } from '../../../quality_indicator'; +const degradedDocsTooltip = ( + + _ignored + + ), + }} + /> +); + +const failedDocsColumnTooltip = ( + +); + // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function Summary() { @@ -32,6 +55,7 @@ export default function Summary() { totalServicesCount, totalHostsCount, totalDegradedDocsCount, + totalFailedDocsCount, quality, } = useOverviewSummaryPanel(); return ( @@ -75,6 +99,13 @@ export default function Summary() { label={overviewPanelDatasetQualityIndicatorDegradedDocs} value={totalDegradedDocsCount} isLoading={isSummaryPanelLoading} + tooltip={degradedDocsTooltip} + /> + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx index e03e0957b5a52..f11bb14be29fd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx @@ -6,7 +6,15 @@ */ import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSkeletonTitle, EuiText } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiPanel, + EuiSkeletonTitle, + EuiText, + EuiToolTip, +} from '@elastic/eui'; import { css } from '@emotion/react'; import { euiThemeVars } from '@kbn/ui-theme'; import { PrivilegesWarningIconWrapper } from '../../../common'; @@ -71,11 +79,13 @@ export function Panel({ export function PanelIndicator({ label, value, + tooltip, isLoading, userHasPrivilege = true, }: { label: string; value: string | number; + tooltip?: React.ReactElement; isLoading: boolean; userHasPrivilege?: boolean; }) { @@ -85,9 +95,23 @@ export function PanelIndicator({ ) : ( <> - - {label} - + {tooltip ? ( + + + {`${label} `} + + + + ) : ( + + {label} + + )} { + const { euiTheme } = useEuiTheme(); + const { + services: { lens }, + } = useKibanaContextForPlugin(); + const { + service, + dataStream, + datasetDetails, + timeRange, + breakdownField, + integrationDetails, + isBreakdownFieldAsserted, + } = useDatasetQualityDetailsState(); + + const { + trackDatasetDetailsBreakdownFieldChanged, + trackDetailsNavigated, + navigationTargets, + navigationSources, + } = useDatasetDetailsTelemetry(); + + const [isChartLoading, setIsChartLoading] = useState(undefined); + const [attributes, setAttributes] = useState | undefined>( + undefined + ); + + const { dataView } = useCreateDataView({ + indexPatternString: getDataViewIndexPattern(dataStream), + }); + + const breakdownDataViewField = useMemo( + () => getDataViewField(dataView, breakdownField), + [breakdownField, dataView] + ); + + const handleChartLoading = (isLoading: boolean) => { + setIsChartLoading(isLoading); + }; + + const handleBreakdownFieldChange = useCallback( + (field: DataViewField | undefined) => { + service.send({ + type: 'BREAKDOWN_FIELD_CHANGE', + breakdownField: field?.name, + }); + }, + [service] + ); + + useEffect(() => { + if (isBreakdownFieldAsserted) trackDatasetDetailsBreakdownFieldChanged(); + }, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]); + + useEffect(() => { + const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW; + const datasetTitle = + integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; + + const lensAttributes = getLensAttributes({ + color: euiTheme.colors.danger, + dataStream: dataStreamName, + datasetTitle, + breakdownFieldName: breakdownDataViewField?.name, + }); + setAttributes(lensAttributes); + }, [ + breakdownDataViewField?.name, + euiTheme.colors.danger, + setAttributes, + dataStream, + integrationDetails?.integration?.datasets, + datasetDetails.name, + ]); + + const openInLensCallback = useCallback(() => { + if (attributes) { + trackDetailsNavigated(navigationTargets.Lens, navigationSources.Chart); + lens.navigateToPrefilledEditor({ + id: '', + timeRange, + attributes, + }); + } + }, [ + attributes, + lens, + navigationSources.Chart, + navigationTargets.Lens, + timeRange, + trackDetailsNavigated, + ]); + + const getOpenInLensAction = useMemo(() => { + return { + id: ACTION_OPEN_IN_LENS, + type: 'link', + order: 17, + getDisplayName(): string { + return openInLensText; + }, + getIconType(): string { + return 'visArea'; + }, + async isCompatible(): Promise { + return true; + }, + async execute(): Promise { + return openInLensCallback(); + }, + }; + }, [openInLensCallback]); + + const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ + query: { language: 'kuery', query: '_ignored:*' }, + navigationSource: navigationSources.Chart, + }); + + const redirectLinkProps = useRedirectLink({ + dataStreamStat: datasetDetails, + query: { language: 'kuery', query: '_ignored:*' }, + timeRangeConfig: timeRange, + breakdownField: breakdownDataViewField?.name, + sendTelemetry, + }); + + const getOpenInLogsExplorerAction = useMemo(() => { + return { + id: ACTION_EXPLORE_IN_LOGS_EXPLORER, + type: 'link', + getDisplayName(): string { + return redirectLinkProps?.isLogsExplorerAvailable + ? exploreDataInLogsExplorerText + : exploreDataInDiscoverText; + }, + getHref: async () => { + return redirectLinkProps.linkProps.href; + }, + getIconType(): string | undefined { + return 'visTable'; + }, + async isCompatible(): Promise { + return true; + }, + async execute(): Promise { + return redirectLinkProps.navigate(); + }, + order: 18, + }; + }, [redirectLinkProps]); + + const extraActions: Action[] = [getOpenInLensAction, getOpenInLogsExplorerAction]; + + const breakdown = useMemo(() => { + return { + dataViewField: breakdownDataViewField, + fieldSupportsBreakdown: breakdownDataViewField + ? fieldSupportsBreakdown(breakdownDataViewField) + : true, + onChange: handleBreakdownFieldChange, + }; + }, [breakdownDataViewField, handleBreakdownFieldChange]); + + return { + attributes, + dataView, + breakdown, + extraActions, + isChartLoading, + onChartLoading: handleChartLoading, + setAttributes, + setIsChartLoading, + }; +}; + +function getDataViewIndexPattern(dataStream: string | undefined) { + return dataStream ?? DEFAULT_LOGS_DATA_VIEW; +} + +function getDataViewField(dataView: DataView | undefined, fieldName: string | undefined) { + return fieldName && dataView + ? dataView.fields.find((field) => field.name === fieldName) + : undefined; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts index 43cf6923075ee..98e7aba30d493 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts @@ -55,12 +55,19 @@ export const useOverviewSummaryPanel = () => { NUMBER_FORMAT ); + const totalFailedDocsCount = formatNumber(dataStreamDetails?.failedDocsCount ?? 0, NUMBER_FORMAT); + const degradedPercentage = - Number(totalDocsCount) > 0 - ? (Number(totalDegradedDocsCount) / Number(totalDocsCount)) * 100 + (dataStreamDetails.docsCount ?? 0) > 0 + ? ((dataStreamDetails?.degradedDocsCount ?? 0) / dataStreamDetails.docsCount!) * 100 + : 0; + + const failedPercentage = + (dataStreamDetails.docsCount ?? 0) > 0 + ? ((dataStreamDetails?.failedDocsCount ?? 0) / dataStreamDetails.docsCount!) * 100 : 0; - const quality = mapPercentageToQuality(degradedPercentage); + const quality = mapPercentageToQuality([degradedPercentage, failedPercentage]); return { totalDocsCount, @@ -70,6 +77,7 @@ export const useOverviewSummaryPanel = () => { totalHostsCount, isSummaryPanelLoading, totalDegradedDocsCount, + totalFailedDocsCount, quality, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 288eff11b92a8..c9212909fcc8c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -21,6 +21,7 @@ import { createDatasetQualityESClient } from '../../../utils'; import { dataStreamService, datasetQualityPrivileges } from '../../../services'; import { getDataStreams } from '../get_data_streams'; import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats'; +import { getFailedDocsPaginated } from '../get_failed_docs'; export async function getDataStreamSettings({ esClient, @@ -92,6 +93,14 @@ export async function getDataStreamDetails({ end ); + const failedDocs = await getFailedDocsPaginated({ + esClient: esClientAsCurrentUser, + types: [], + datasetQuery: dataStream, + start, + end, + }); + const avgDocSizeInBytes = hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0 ? isServerless @@ -103,6 +112,7 @@ export async function getDataStreamDetails({ return { ...dataStreamSummaryStats, + failedDocsCount: failedDocs[0]?.count, sizeBytes, lastActivity: esDataStream?.lastActivity, userPrivileges: { diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts new file mode 100644 index 0000000000000..170da8def9439 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { APIClientRequestParamsOf } from '@kbn/dataset-quality-plugin/common/rest'; +import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { merge } from 'lodash'; +import rison from '@kbn/rison'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('synthtrace'); + const from = '2024-09-20T11:00:00.000Z'; + const to = '2024-09-20T11:01:00.000Z'; + const dataStreamType = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${dataStreamType}-${dataset}-${namespace}`; + const syntheticsDataStreamName = `${dataStreamType}-${syntheticsDataset}-${namespace}`; + + const endpoint = 'GET /internal/dataset_quality/data_streams/failed_docs'; + type ApiParams = APIClientRequestParamsOf['params']['query']; + type DataStreamType = ApiParams['types'][0]; + + const processors = [ + { + script: { + tag: 'normalize log level', + lang: 'painless', + source: ` + String level = ctx['log.level']; + if ('0'.equals(level)) { + ctx['log.level'] = 'info'; + } else if ('1'.equals(level)) { + ctx['log.level'] = 'debug'; + } else if ('2'.equals(level)) { + ctx['log.level'] = 'warning'; + } else if ('3'.equals(level)) { + ctx['log.level'] = 'error'; + } else { + throw new Exception("Not a valid log level"); + } + `, + }, + }, + ]; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { types = rison.encodeArray(['logs']), start, end }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: Omit & { types?: DataStreamType[] }; + }) { + return roleScopedSupertestWithCookieCredentials + .get(`/internal/dataset_quality/data_streams/failed_docs`) + .query({ + types, + start, + end, + }); + } + + describe('DataStream failed docs', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + let synthtraceLogsEsClient: LogsSynthtraceEsClient; + + before(async () => { + synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient(); + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + + await synthtraceLogsEsClient.createCustomPipeline(processors); + await synthtraceLogsEsClient.updateIndexTemplate('logs', (template) => { + const next = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + }); + await synthtraceLogsEsClient.index([ + timerange(from, to) + .interval('1m') + .rate(1) + .generator((timestamp) => [ + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .logLevel('0') + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .logLevel('5') + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + ]), + ]); + }); + + after(async () => { + await synthtraceLogsEsClient.clean(); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('returns number of failed documents per DataStream', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: from, + end: to, + }, + }); + + expect(resp.body.failedDocs.length).to.be(1); + expect(resp.body.totalDocs[0].dataset).to.be(syntheticsDataStreamName); + expect(resp.body.totalDocs[0].count).to.be(1); + }); + + it('returns empty when all documents are outside timeRange', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: '2024-09-21T11:00:00.000Z', + end: '2024-09-21T11:01:00.000Z', + }, + }); + + expect(resp.body.failedDocs.length).to.be(0); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0481e882aee6e..5f9ce787a44dd 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -19,5 +19,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./degraded_fields')); loadTestFile(require.resolve('./data_stream_details')); loadTestFile(require.resolve('./degraded_field_values')); + loadTestFile(require.resolve('./failed_docs')); }); } From 91e2c3ec0bc9535f2fcee86b25bcb8b72c816adf Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 18 Nov 2024 15:48:15 -0300 Subject: [PATCH 06/19] Addresing Pr cmments --- .../common/utils/dataset_name.ts | 1 + .../dataset_quality/table/columns.tsx | 10 ++- .../failed_docs/lens_attributes.ts | 9 +-- .../{degraded_docs => }/index.tsx | 71 +++++++++++-------- ...ed_docs_chart.tsx => trend_docs_chart.tsx} | 14 ++-- .../overview/index.tsx | 4 +- .../hooks/use_dataset_details_telemetry.ts | 4 +- .../hooks/use_dataset_quality_table.tsx | 14 +++- .../dataset_quality_controller/src/types.ts | 8 +++ .../routes/data_streams/get_failed_docs.ts | 1 + .../utils/create_dataset_quality_es_client.ts | 1 + 11 files changed, 89 insertions(+), 48 deletions(-) rename x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/{degraded_docs => }/index.tsx (71%) rename x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/{degraded_docs/degraded_docs_chart.tsx => trend_docs_chart.tsx} (88%) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts index 08944df9efcdc..5cf347ed09304 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts @@ -40,6 +40,7 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => { }; export const extractIndexNameFromBackingIndex = (indexString: string): string => { + // TODO: Undo this change once ::failures is supported const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; const match = indexString.match(pattern); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 5e8b7d080702b..a12f9767be048 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -171,7 +171,9 @@ export const getDatasetQualityTableColumns = ({ canUserMonitorDataset, canUserMonitorAnyDataStream, loadingDataStreamStats, + loadingDocStats, loadingDegradedStats, + loadingFailedStats, showFullDatasetNames, isActiveDataset, timeRange, @@ -181,7 +183,9 @@ export const getDatasetQualityTableColumns = ({ canUserMonitorDataset: boolean; canUserMonitorAnyDataStream: boolean; loadingDataStreamStats: boolean; + loadingDocStats: boolean; loadingDegradedStats: boolean; + loadingFailedStats: boolean; showFullDatasetNames: boolean; isActiveDataset: (lastActivity: number) => boolean; timeRange: TimeRangeConfig; @@ -260,7 +264,7 @@ export const getDatasetQualityTableColumns = ({ width="60px" height="20px" borderRadius="m" - isLoading={loadingDataStreamStats || loadingDegradedStats} + isLoading={loadingDataStreamStats || loadingDegradedStats || loadingDocStats} > {formatNumber( DataStreamStat.calculateFilteredSize(dataStreamStat), @@ -289,7 +293,7 @@ export const getDatasetQualityTableColumns = ({ sortable: true, render: (_, dataStreamStat: DataStreamStat) => ( ), @@ -332,7 +336,7 @@ export const getDatasetQualityTableColumns = ({ sortable: true, render: (_, dataStreamStat: DataStreamStat) => ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts index 1479a98b799e3..4e76999318f57 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts @@ -130,6 +130,7 @@ export function getLensAttributes({ } function getAdHocDataViewState(id: string, dataStream: string, title: string) { + // TODO: Need to fix the index pattern used here (aka ::failures) return { internalReferences: [ { @@ -244,7 +245,7 @@ function getChartColumns(breakdownField?: string): Record - i18n.translate('xpack.datasetQuality.details.degradedDocsTopNValues', { +const getFlyoutFailedDocsTopNText = (count: number, fieldName: string) => + i18n.translate('xpack.datasetQuality.details.failedDocsTopNValues', { defaultMessage: 'Top {count} values of {fieldName}', values: { count, fieldName }, description: - 'Tooltip label for the top N values of a field in the degraded documents trend chart.', + 'Tooltip label for the top N values of a field in the failed documents trend chart.', }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx similarity index 71% rename from x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx rename to x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx index 347c970577446..5039c67bcd668 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx @@ -25,23 +25,24 @@ import { import type { DataViewField } from '@kbn/data-views-plugin/common'; import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; -import { useFailedDocsChart } from '../../../../../hooks/use_failed_docs_chart'; +import { i18n } from '@kbn/i18n'; +import { useFailedDocsChart } from '../../../../hooks/use_failed_docs_chart'; import { discoverAriaText, logsExplorerAriaText, openInDiscoverText, openInLogsExplorerText, overviewTrendsDocsText, -} from '../../../../../../common/translations'; -import { DegradedDocsChart } from './degraded_docs_chart'; +} from '../../../../../common/translations'; +import { TrendDocsChart } from './trend_docs_chart'; import { useDatasetDetailsRedirectLinkTelemetry, useDatasetQualityDetailsState, useDegradedDocsChart, useRedirectLink, -} from '../../../../../hooks'; -import { _IGNORED } from '../../../../../../common/es_fields'; -import { NavigationSource } from '../../../../../services/telemetry'; +} from '../../../../hooks'; +import { _IGNORED } from '../../../../../common/es_fields'; +import { NavigationSource } from '../../../../services/telemetry'; const trendDocsTooltip = ( - {selectedChart === 'degradedDocs' ? ( - - - - ) : ( - <> - )} + + + {selectedChart === 'degradedDocs' ? ( - ) : ( - , 'attributes' | 'isChartLoading' | 'onChartLoading' | 'extraActions' @@ -34,7 +34,7 @@ interface DegradedDocsChartProps onTimeRangeChange: (props: Pick) => void; } -export function DegradedDocsChart({ +export function TrendDocsChart({ attributes, isChartLoading, onChartLoading, @@ -42,7 +42,7 @@ export function DegradedDocsChart({ timeRange, lastReloadTime, onTimeRangeChange, -}: DegradedDocsChartProps) { +}: TrendDocsChartProps) { const { services: { lens }, } = useKibanaContextForPlugin(); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx index 380dd6bf09b95..456a43f5b83bf 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/index.tsx @@ -14,7 +14,7 @@ import { DegradedFields } from './degraded_fields'; const OverviewHeader = dynamic(() => import('./header')); const Summary = dynamic(() => import('./summary')); -const DegradedDocs = dynamic(() => import('./document_trends/degraded_docs')); +const DocumentTrends = dynamic(() => import('./document_trends')); export function Overview() { const { dataStream, isNonAggregatable, updateTimeRange } = useDatasetQualityDetailsState(); @@ -34,7 +34,7 @@ export function Overview() { - + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts index e98900f77ae44..13f0dce03debc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts @@ -167,10 +167,12 @@ function getDatasetDetailsEbtProps({ type: datasetDetails.type, }; const degradedDocs = dataStreamDetails?.degradedDocsCount ?? 0; + const failedDocs = dataStreamDetails?.failedDocsCount ?? 0; const totalDocs = dataStreamDetails?.docsCount ?? 0; const degradedPercentage = totalDocs > 0 ? Number(((degradedDocs / totalDocs) * 100).toFixed(2)) : 0; - const health = mapPercentageToQuality([degradedPercentage]); + const failedPercentage = totalDocs > 0 ? Number(((failedDocs / totalDocs) * 100).toFixed(2)) : 0; + const health = mapPercentageToQuality([degradedPercentage, failedPercentage]); const { startDate: from, endDate: to } = getDateISORange(timeRange); return { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 30ff971043986..e61706bdf9813 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -66,15 +66,23 @@ export const useDatasetQualityTable = () => { service, (state) => state.matches('stats.datasets.fetching') || + state.matches('stats.docsStats.fetching') || state.matches('integrations.fetching') || - state.matches('stats.degradedDocs.fetching') + state.matches('stats.degradedDocs.fetching') || + state.matches('stats.failedDocs.fetching') ); const loadingDataStreamStats = useSelector(service, (state) => state.matches('stats.datasets.fetching') ); + const loadingDocStats = useSelector(service, (state) => + state.matches('stats.docsStats.fetching') + ); const loadingDegradedStats = useSelector(service, (state) => state.matches('stats.degradedDocs.fetching') ); + const loadingFailedStats = useSelector(service, (state) => + state.matches('stats.failedDocs.fetching') + ); const datasets = useSelector(service, (state) => state.context.datasets); @@ -100,7 +108,9 @@ export const useDatasetQualityTable = () => { canUserMonitorDataset, canUserMonitorAnyDataStream, loadingDataStreamStats, + loadingDocStats, loadingDegradedStats, + loadingFailedStats, showFullDatasetNames, isActiveDataset: isActive, timeRange, @@ -111,7 +121,9 @@ export const useDatasetQualityTable = () => { canUserMonitorDataset, canUserMonitorAnyDataStream, loadingDataStreamStats, + loadingDocStats, loadingDegradedStats, + loadingFailedStats, showFullDatasetNames, isActive, timeRange, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 4f3c6716fcf24..9d71984ec51d0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -97,10 +97,18 @@ export type DatasetQualityControllerTypeState = value: 'stats.datasets.loaded'; context: DefaultDatasetQualityStateContext; } + | { + value: 'stats.docsStats.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'stats.degradedDocs.fetching'; context: DefaultDatasetQualityStateContext; } + | { + value: 'stats.failedDocs.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'stats.nonAggregatableDatasets.fetching'; context: DefaultDatasetQualityStateContext; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts index ca93bd8c01d6b..199401ac8e3a0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_failed_docs.ts @@ -29,6 +29,7 @@ async function getPaginatedResults(options: { filter: [...rangeQuery(start, end)], }; + // TODO: Fix index for accesing failure store (::failures) and remove the search parameter const response = await datasetQualityESClient.search({ index, size: 0, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 24a96ab9b2911..53db8217d18d6 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -22,6 +22,7 @@ import { type DatasetQualityESSearchParams = ESSearchRequest & { size: number; + // TODO: Remove search parameter once ::failures is supported failure_store?: 'only'; }; From f753581a92c29356e21db037a45445422ed5f2b2 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 19 Nov 2024 12:12:42 -0300 Subject: [PATCH 07/19] extracting percentage calculator to an util --- .../common/data_streams_stats/data_stream_stat.ts | 4 ---- .../components/dataset_quality/table/columns.tsx | 2 +- .../public/hooks/use_dataset_details_telemetry.ts | 6 +++--- .../public/utils/calculate_percentage.ts | 10 ++++++++++ .../dataset_quality/public/utils/generate_datasets.ts | 5 +++-- .../dataset_quality/public/utils/index.ts | 1 + 6 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 40fa5d682b8bd..d9673d0788921 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -118,8 +118,4 @@ export class DataStreamStat { const avgDocSize = sizeBytes && totalDocs ? sizeBytes / totalDocs : 0; return avgDocSize * (docsInTimeRange ?? 0); } - - public static calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) { - return totalDocs && count ? (count / totalDocs) * 100 : 0; - } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index a12f9767be048..b6ddbbd254568 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -264,7 +264,7 @@ export const getDatasetQualityTableColumns = ({ width="60px" height="20px" borderRadius="m" - isLoading={loadingDataStreamStats || loadingDegradedStats || loadingDocStats} + isLoading={loadingDataStreamStats || loadingDocStats} > {formatNumber( DataStreamStat.calculateFilteredSize(dataStreamStat), diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts index 13f0dce03debc..2e88af3acf06d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts @@ -15,6 +15,7 @@ import { DataStreamDetails } from '../../common/api_types'; import { Integration } from '../../common/data_streams_stats/integration'; import { mapPercentageToQuality } from '../../common/utils'; import { MASKED_FIELD_PLACEHOLDER, UNKOWN_FIELD_PLACEHOLDER } from '../../common/constants'; +import { calculatePercentage } from '../utils'; export function useDatasetDetailsTelemetry() { const { @@ -169,9 +170,8 @@ function getDatasetDetailsEbtProps({ const degradedDocs = dataStreamDetails?.degradedDocsCount ?? 0; const failedDocs = dataStreamDetails?.failedDocsCount ?? 0; const totalDocs = dataStreamDetails?.docsCount ?? 0; - const degradedPercentage = - totalDocs > 0 ? Number(((degradedDocs / totalDocs) * 100).toFixed(2)) : 0; - const failedPercentage = totalDocs > 0 ? Number(((failedDocs / totalDocs) * 100).toFixed(2)) : 0; + const degradedPercentage = calculatePercentage({ totalDocs, count: degradedDocs }); + const failedPercentage = calculatePercentage({ totalDocs, count: failedDocs }); const health = mapPercentageToQuality([degradedPercentage, failedPercentage]); const { startDate: from, endDate: to } = getDateISORange(timeRange); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts new file mode 100644 index 0000000000000..a320d5d4641d7 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/calculate_percentage.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) { + return totalDocs && count ? Number(((count / totalDocs) * 100).toFixed(2)) : 0; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 2efeb34d42da7..3e170a493e7e1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -13,6 +13,7 @@ import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; import { DictionaryType } from '../state_machines/dataset_quality_controller/src/types'; import { flattenStats } from './flatten_stats'; +import { calculatePercentage } from './calculate_percentage'; export function generateDatasets( dataStreamStats: DataStreamStatType[] = [], @@ -64,7 +65,7 @@ export function generateDatasets( Object.assign(degradedMapAcc, { [dataset]: { count, - percentage: DataStreamStat.calculatePercentage({ + percentage: calculatePercentage({ totalDocs: totalDocsMap[dataset], count, }), @@ -84,7 +85,7 @@ export function generateDatasets( Object.assign(failedMapAcc, { [dataset]: { count, - percentage: DataStreamStat.calculatePercentage({ + percentage: calculatePercentage({ totalDocs: totalDocsMap[dataset] ? totalDocsMap[dataset] + count : 0, count, }), diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts index 3185367e39aca..fe70b6e737a27 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +export * from './calculate_percentage'; export * from './filter_inactive_datasets'; export * from './generate_datasets'; export * from './use_kibana'; From 74df05ca0b9318210f595511236bdf959bf367e9 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 19 Nov 2024 16:01:17 -0300 Subject: [PATCH 08/19] merging useDocsChart hooks for quality issues + removing unnecesary useEffect --- .../overview/document_trends/index.tsx | 108 +++------ .../document_trends/trend_docs_chart.tsx | 4 +- .../dataset_quality/public/hooks/index.ts | 2 +- .../public/hooks/use_failed_docs_chart.ts | 227 ------------------ ...rt.ts => use_quality_issues_docs_chart.ts} | 44 +++- 5 files changed, 62 insertions(+), 323 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_failed_docs_chart.ts rename x-pack/plugins/observability_solution/dataset_quality/public/hooks/{use_degraded_docs_chart.ts => use_quality_issues_docs_chart.ts} (80%) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx index 5039c67bcd668..ec6b25e32aec3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, @@ -22,11 +22,9 @@ import { OnTimeChangeProps, useGeneratedHtmlId, } from '@elastic/eui'; -import type { DataViewField } from '@kbn/data-views-plugin/common'; import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; import { i18n } from '@kbn/i18n'; -import { useFailedDocsChart } from '../../../../hooks/use_failed_docs_chart'; import { discoverAriaText, logsExplorerAriaText, @@ -36,13 +34,10 @@ import { } from '../../../../../common/translations'; import { TrendDocsChart } from './trend_docs_chart'; import { - useDatasetDetailsRedirectLinkTelemetry, + QualityIssue, useDatasetQualityDetailsState, - useDegradedDocsChart, - useRedirectLink, + useQualityIssuesDocsChart, } from '../../../../hooks'; -import { _IGNORED } from '../../../../../common/es_fields'; -import { NavigationSource } from '../../../../services/telemetry'; const trendDocsTooltip = ( ); -const DEGRADED_DOCS_KUERY = `${_IGNORED}: *`; - // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: number }) { - const { timeRange, updateTimeRange, datasetDetails } = useDatasetQualityDetailsState(); - const { - dataView: degradedDataView, - breakdown: degradedBreakdown, - ...degradeChartProps - } = useDegradedDocsChart(); - const { - dataView: failedDataView, - breakdown: failedBreakDown, - ...failedChartProps - } = useFailedDocsChart(); + const [selectedChart, setSelectedChart] = useState('degradedDocs'); + + const { timeRange, updateTimeRange } = useDatasetQualityDetailsState(); + const { dataView, breakdown, redirectLinkProps, ...qualityIssuesChartProps } = + useQualityIssuesDocsChart(selectedChart); const accordionId = useGeneratedHtmlId({ prefix: overviewTrendsDocsText, }); - const [breakdownDataViewField, setBreakdownDataViewField] = useState( - undefined - ); - - const [selectedChart, setSelectedChart] = useState('degradedDocs'); - - const query = selectedChart === 'degradedDocs' ? DEGRADED_DOCS_KUERY : ''; - - const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ - query: { language: 'kuery', query }, - navigationSource: NavigationSource.Trend, - }); - - const qualityIssueDocsLinkLogsExplorer = useRedirectLink({ - dataStreamStat: datasetDetails, - timeRangeConfig: timeRange, - query: { - language: 'kuery', - query, - }, - sendTelemetry, - }); - - useEffect(() => { - if (degradedBreakdown.dataViewField && degradedBreakdown.fieldSupportsBreakdown) { - setBreakdownDataViewField(degradedBreakdown.dataViewField); - } else { - setBreakdownDataViewField(undefined); - } - }, [setBreakdownDataViewField, degradedBreakdown]); - const onTimeRangeChange = useCallback( ({ start, end }: Pick) => { updateTimeRange({ start, end, refreshInterval: timeRange.refresh.value }); @@ -144,7 +100,7 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num legend={i18n.translate('xpack.datasetQuality.details.chartTypeLegend', { defaultMessage: 'Quality chart type', })} - onChange={setSelectedChart} + onChange={(id) => setSelectedChart(id as QualityIssue)} options={[ { id: 'degradedDocs', @@ -162,16 +118,21 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num idSelected={selectedChart} /> - + - {selectedChart === 'degradedDocs' ? ( - - ) : ( - - )} + ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/trend_docs_chart.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/trend_docs_chart.tsx index b91f481f038dd..e3b77366908ca 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/trend_docs_chart.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/trend_docs_chart.tsx @@ -14,7 +14,7 @@ import { KibanaErrorBoundary } from '@kbn/shared-ux-error-boundary'; import { flyoutDegradedDocsTrendText } from '../../../../../common/translations'; import { useKibanaContextForPlugin } from '../../../../utils'; import { TimeRangeConfig } from '../../../../../common/types'; -import { useDegradedDocsChart } from '../../../../hooks'; +import { useQualityIssuesDocsChart } from '../../../../hooks'; const CHART_HEIGHT = 180; const DISABLED_ACTIONS = [ @@ -26,7 +26,7 @@ const DISABLED_ACTIONS = [ interface TrendDocsChartProps extends Pick< - ReturnType, + ReturnType, 'attributes' | 'isChartLoading' | 'onChartLoading' | 'extraActions' > { timeRange: TimeRangeConfig; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts index ad588fd0b673f..59c3480a24518 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts @@ -6,7 +6,7 @@ */ export * from './use_dataset_quality_table'; -export * from './use_degraded_docs_chart'; +export * from './use_quality_issues_docs_chart'; export * from './use_redirect_link'; export * from './use_summary_panel'; export * from './use_create_dataview'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_failed_docs_chart.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_failed_docs_chart.ts deleted file mode 100644 index 275ac903b92c6..0000000000000 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_failed_docs_chart.ts +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useCallback, useEffect, useMemo, useState } from 'react'; -import type { Action } from '@kbn/ui-actions-plugin/public'; -import { i18n } from '@kbn/i18n'; -import { useEuiTheme } from '@elastic/eui'; -import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; -import { fieldSupportsBreakdown } from '@kbn/field-utils'; -import { DEFAULT_LOGS_DATA_VIEW } from '../../common/constants'; -import { useCreateDataView } from './use_create_dataview'; -import { useKibanaContextForPlugin } from '../utils'; -import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state'; -import { getLensAttributes } from '../components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes'; -import { useRedirectLink } from './use_redirect_link'; -import { useDatasetDetailsTelemetry } from './use_dataset_details_telemetry'; -import { useDatasetDetailsRedirectLinkTelemetry } from './use_redirect_link_telemetry'; - -const exploreDataInLogsExplorerText = i18n.translate( - 'xpack.datasetQuality.details.chartExploreDataInLogsExplorerText', - { - defaultMessage: 'Explore data in Logs Explorer', - } -); - -const exploreDataInDiscoverText = i18n.translate( - 'xpack.datasetQuality.details.chartExploreDataInDiscoverText', - { - defaultMessage: 'Explore data in Discover', - } -); - -const openInLensText = i18n.translate('xpack.datasetQuality.details.chartOpenInLensText', { - defaultMessage: 'Open in Lens', -}); - -const ACTION_EXPLORE_IN_LOGS_EXPLORER = 'ACTION_EXPLORE_IN_LOGS_EXPLORER'; -const ACTION_OPEN_IN_LENS = 'ACTION_OPEN_IN_LENS'; - -export const useFailedDocsChart = () => { - const { euiTheme } = useEuiTheme(); - const { - services: { lens }, - } = useKibanaContextForPlugin(); - const { - service, - dataStream, - datasetDetails, - timeRange, - breakdownField, - integrationDetails, - isBreakdownFieldAsserted, - } = useDatasetQualityDetailsState(); - - const { - trackDatasetDetailsBreakdownFieldChanged, - trackDetailsNavigated, - navigationTargets, - navigationSources, - } = useDatasetDetailsTelemetry(); - - const [isChartLoading, setIsChartLoading] = useState(undefined); - const [attributes, setAttributes] = useState | undefined>( - undefined - ); - - const { dataView } = useCreateDataView({ - indexPatternString: getDataViewIndexPattern(dataStream), - }); - - const breakdownDataViewField = useMemo( - () => getDataViewField(dataView, breakdownField), - [breakdownField, dataView] - ); - - const handleChartLoading = (isLoading: boolean) => { - setIsChartLoading(isLoading); - }; - - const handleBreakdownFieldChange = useCallback( - (field: DataViewField | undefined) => { - service.send({ - type: 'BREAKDOWN_FIELD_CHANGE', - breakdownField: field?.name, - }); - }, - [service] - ); - - useEffect(() => { - if (isBreakdownFieldAsserted) trackDatasetDetailsBreakdownFieldChanged(); - }, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]); - - useEffect(() => { - const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW; - const datasetTitle = - integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; - - const lensAttributes = getLensAttributes({ - color: euiTheme.colors.danger, - dataStream: dataStreamName, - datasetTitle, - breakdownFieldName: breakdownDataViewField?.name, - }); - setAttributes(lensAttributes); - }, [ - breakdownDataViewField?.name, - euiTheme.colors.danger, - setAttributes, - dataStream, - integrationDetails?.integration?.datasets, - datasetDetails.name, - ]); - - const openInLensCallback = useCallback(() => { - if (attributes) { - trackDetailsNavigated(navigationTargets.Lens, navigationSources.Chart); - lens.navigateToPrefilledEditor({ - id: '', - timeRange, - attributes, - }); - } - }, [ - attributes, - lens, - navigationSources.Chart, - navigationTargets.Lens, - timeRange, - trackDetailsNavigated, - ]); - - const getOpenInLensAction = useMemo(() => { - return { - id: ACTION_OPEN_IN_LENS, - type: 'link', - order: 17, - getDisplayName(): string { - return openInLensText; - }, - getIconType(): string { - return 'visArea'; - }, - async isCompatible(): Promise { - return true; - }, - async execute(): Promise { - return openInLensCallback(); - }, - }; - }, [openInLensCallback]); - - const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ - query: { language: 'kuery', query: '_ignored:*' }, - navigationSource: navigationSources.Chart, - }); - - const redirectLinkProps = useRedirectLink({ - dataStreamStat: datasetDetails, - query: { language: 'kuery', query: '_ignored:*' }, - timeRangeConfig: timeRange, - breakdownField: breakdownDataViewField?.name, - sendTelemetry, - }); - - const getOpenInLogsExplorerAction = useMemo(() => { - return { - id: ACTION_EXPLORE_IN_LOGS_EXPLORER, - type: 'link', - getDisplayName(): string { - return redirectLinkProps?.isLogsExplorerAvailable - ? exploreDataInLogsExplorerText - : exploreDataInDiscoverText; - }, - getHref: async () => { - return redirectLinkProps.linkProps.href; - }, - getIconType(): string | undefined { - return 'visTable'; - }, - async isCompatible(): Promise { - return true; - }, - async execute(): Promise { - return redirectLinkProps.navigate(); - }, - order: 18, - }; - }, [redirectLinkProps]); - - const extraActions: Action[] = [getOpenInLensAction, getOpenInLogsExplorerAction]; - - const breakdown = useMemo(() => { - return { - dataViewField: breakdownDataViewField, - fieldSupportsBreakdown: breakdownDataViewField - ? fieldSupportsBreakdown(breakdownDataViewField) - : true, - onChange: handleBreakdownFieldChange, - }; - }, [breakdownDataViewField, handleBreakdownFieldChange]); - - return { - attributes, - dataView, - breakdown, - extraActions, - isChartLoading, - onChartLoading: handleChartLoading, - setAttributes, - setIsChartLoading, - }; -}; - -function getDataViewIndexPattern(dataStream: string | undefined) { - return dataStream ?? DEFAULT_LOGS_DATA_VIEW; -} - -function getDataViewField(dataView: DataView | undefined, fieldName: string | undefined) { - return fieldName && dataView - ? dataView.fields.find((field) => field.name === fieldName) - : undefined; -} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts similarity index 80% rename from x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts rename to x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts index 90779613f0c06..328ffb2cebf42 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts @@ -15,7 +15,8 @@ import { DEFAULT_LOGS_DATA_VIEW } from '../../common/constants'; import { useCreateDataView } from './use_create_dataview'; import { useKibanaContextForPlugin } from '../utils'; import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state'; -import { getLensAttributes } from '../components/dataset_quality_details/overview/document_trends/degraded_docs/lens_attributes'; +import { getLensAttributes as getDegradedLensAttributes } from '../components/dataset_quality_details/overview/document_trends/degraded_docs/lens_attributes'; +import { getLensAttributes as getFailedLensAttributes } from '../components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes'; import { useRedirectLink } from './use_redirect_link'; import { useDatasetDetailsTelemetry } from './use_dataset_details_telemetry'; import { useDatasetDetailsRedirectLinkTelemetry } from './use_redirect_link_telemetry'; @@ -40,8 +41,11 @@ const openInLensText = i18n.translate('xpack.datasetQuality.details.chartOpenInL const ACTION_EXPLORE_IN_LOGS_EXPLORER = 'ACTION_EXPLORE_IN_LOGS_EXPLORER'; const ACTION_OPEN_IN_LENS = 'ACTION_OPEN_IN_LENS'; +const DEGRADED_DOCS_KUERY = `_ignored:*`; -export const useDegradedDocsChart = () => { +export type QualityIssue = 'degradedDocs' | 'failedDocs'; + +export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { const { euiTheme } = useEuiTheme(); const { services: { lens }, @@ -64,9 +68,11 @@ export const useDegradedDocsChart = () => { } = useDatasetDetailsTelemetry(); const [isChartLoading, setIsChartLoading] = useState(undefined); - const [attributes, setAttributes] = useState | undefined>( - undefined - ); + const [attributes, setAttributes] = useState< + ReturnType | undefined + >(undefined); + + const query = qualityIssue === 'degradedDocs' ? DEGRADED_DOCS_KUERY : ''; const { dataView } = useCreateDataView({ indexPatternString: getDataViewIndexPattern(dataStream), @@ -96,18 +102,28 @@ export const useDegradedDocsChart = () => { }, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]); useEffect(() => { + // TODO: Fix dataStreamName for accesing failure store (::failures) const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW; const datasetTitle = integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; - const lensAttributes = getLensAttributes({ - color: euiTheme.colors.danger, - dataStream: dataStreamName, - datasetTitle, - breakdownFieldName: breakdownDataViewField?.name, - }); + const lensAttributes = + qualityIssue === 'degradedDocs' + ? getDegradedLensAttributes({ + color: euiTheme.colors.danger, + dataStream: dataStreamName, + datasetTitle, + breakdownFieldName: breakdownDataViewField?.name, + }) + : getFailedLensAttributes({ + color: euiTheme.colors.danger, + dataStream: dataStreamName, + datasetTitle, + breakdownFieldName: breakdownDataViewField?.name, + }); setAttributes(lensAttributes); }, [ + qualityIssue, breakdownDataViewField?.name, euiTheme.colors.danger, setAttributes, @@ -155,13 +171,13 @@ export const useDegradedDocsChart = () => { }, [openInLensCallback]); const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ - query: { language: 'kuery', query: '_ignored:*' }, + query: { language: 'kuery', query }, navigationSource: navigationSources.Chart, }); const redirectLinkProps = useRedirectLink({ dataStreamStat: datasetDetails, - query: { language: 'kuery', query: '_ignored:*' }, + query: { language: 'kuery', query }, timeRangeConfig: timeRange, breakdownField: breakdownDataViewField?.name, sendTelemetry, @@ -210,12 +226,14 @@ export const useDegradedDocsChart = () => { breakdown, extraActions, isChartLoading, + redirectLinkProps, onChartLoading: handleChartLoading, setAttributes, setIsChartLoading, }; }; +// TODO: Fix dataView for accesing failure store (::failures) function getDataViewIndexPattern(dataStream: string | undefined) { return dataStream ?? DEFAULT_LOGS_DATA_VIEW; } From 73a90be13368d9917364e71b65ac901ab8ea2649 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 19 Nov 2024 16:39:26 -0300 Subject: [PATCH 09/19] Added qualityIssuesChart to the store + syncing URL --- .../data_quality/common/url_schema/common.ts | 5 ++++ .../dataset_quality_details_url_schema_v1.ts | 3 ++- .../dataset_quality_details/url_schema_v1.ts | 2 ++ .../overview/document_trends/index.tsx | 26 +++++++++---------- .../dataset_quality_details/public_state.ts | 2 ++ .../dataset_quality_details/types.ts | 6 ++++- .../use_dataset_quality_details_state.ts | 3 +++ .../hooks/use_quality_issues_docs_chart.ts | 23 +++++++++++----- .../defaults.ts | 1 + .../state_machine.ts | 9 +++++++ .../types.ts | 13 +++++++++- 11 files changed, 71 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/data_quality/common/url_schema/common.ts b/x-pack/plugins/data_quality/common/url_schema/common.ts index eb929faee1b00..bd7a271a5b778 100644 --- a/x-pack/plugins/data_quality/common/url_schema/common.ts +++ b/x-pack/plugins/data_quality/common/url_schema/common.ts @@ -9,6 +9,11 @@ import * as rt from 'io-ts'; export const DATA_QUALITY_URL_STATE_KEY = 'pageState'; +export const qualityIssuesRT = rt.keyof({ + degradedDocs: null, + failedDocs: null, +}); + export const directionRT = rt.keyof({ asc: null, desc: null, diff --git a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts index 97c7771bbf994..deba69f697245 100644 --- a/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts +++ b/x-pack/plugins/data_quality/common/url_schema/dataset_quality_details_url_schema_v1.ts @@ -6,7 +6,7 @@ */ import * as rt from 'io-ts'; -import { dataStreamRT, degradedFieldRT, timeRangeRT } from './common'; +import { dataStreamRT, degradedFieldRT, qualityIssuesRT, timeRangeRT } from './common'; export const urlSchemaRT = rt.exact( rt.intersection([ @@ -16,6 +16,7 @@ export const urlSchemaRT = rt.exact( rt.partial({ v: rt.literal(1), timeRange: timeRangeRT, + qualityIssuesChart: qualityIssuesRT, breakdownField: rt.string, degradedFields: degradedFieldRT, expandedDegradedField: rt.string, diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts index 7b91895598eca..30a65303aec5a 100644 --- a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts +++ b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/url_schema_v1.ts @@ -17,6 +17,7 @@ export const getStateFromUrlValue = ( dataStream: urlValue.dataStream, timeRange: urlValue.timeRange, degradedFields: urlValue.degradedFields, + qualityIssuesChart: urlValue.qualityIssuesChart, breakdownField: urlValue.breakdownField, expandedDegradedField: urlValue.expandedDegradedField, showCurrentQualityIssues: urlValue.showCurrentQualityIssues, @@ -30,6 +31,7 @@ export const getUrlValueFromState = ( timeRange: state.timeRange, degradedFields: state.degradedFields, breakdownField: state.breakdownField, + qualityIssuesChart: state.qualityIssuesChart, expandedDegradedField: state.expandedDegradedField, showCurrentQualityIssues: state.showCurrentQualityIssues, v: 1, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx index ec6b25e32aec3..541a6bee711a0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useState } from 'react'; +import React, { useCallback } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, @@ -25,6 +25,7 @@ import { import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; import { i18n } from '@kbn/i18n'; +import { QualityIssue } from '../../../../state_machines/dataset_quality_details_controller'; import { discoverAriaText, logsExplorerAriaText, @@ -33,11 +34,7 @@ import { overviewTrendsDocsText, } from '../../../../../common/translations'; import { TrendDocsChart } from './trend_docs_chart'; -import { - QualityIssue, - useDatasetQualityDetailsState, - useQualityIssuesDocsChart, -} from '../../../../hooks'; +import { useDatasetQualityDetailsState, useQualityIssuesDocsChart } from '../../../../hooks'; const trendDocsTooltip = ( ('degradedDocs'); - - const { timeRange, updateTimeRange } = useDatasetQualityDetailsState(); - const { dataView, breakdown, redirectLinkProps, ...qualityIssuesChartProps } = - useQualityIssuesDocsChart(selectedChart); + const { timeRange, updateTimeRange, docsTrendChart } = useDatasetQualityDetailsState(); + const { + dataView, + breakdown, + redirectLinkProps, + handleDocsTrendChartChange, + ...qualityIssuesChartProps + } = useQualityIssuesDocsChart(); const accordionId = useGeneratedHtmlId({ prefix: overviewTrendsDocsText, @@ -100,7 +100,7 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num legend={i18n.translate('xpack.datasetQuality.details.chartTypeLegend', { defaultMessage: 'Quality chart type', })} - onChange={(id) => setSelectedChart(id as QualityIssue)} + onChange={(id) => handleDocsTrendChartChange(id as QualityIssue)} options={[ { id: 'degradedDocs', @@ -115,7 +115,7 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num }), }, ]} - idSelected={selectedChart} + idSelected={docsTrendChart} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts index a87712a5e364e..4d82c763c7c20 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/public_state.ts @@ -20,6 +20,7 @@ export const getPublicStateFromContext = ( degradedFields: context.degradedFields, timeRange: context.timeRange, breakdownField: context.breakdownField, + qualityIssuesChart: context.qualityIssuesChart, integration: context.integration, expandedDegradedField: context.expandedDegradedField, showCurrentQualityIssues: context.showCurrentQualityIssues, @@ -51,6 +52,7 @@ export const getContextFromPublicState = ( }, }, dataStream: publicState.dataStream, + qualityIssuesChart: publicState.qualityIssuesChart ?? DEFAULT_CONTEXT.qualityIssuesChart, breakdownField: publicState.breakdownField, expandedDegradedField: publicState.expandedDegradedField, showCurrentQualityIssues: diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts index 1f9397ee4504c..0e37c2fad8047 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/dataset_quality_details/types.ts @@ -30,7 +30,11 @@ export type DatasetQualityDetailsPublicState = WithDefaultControllerState; export type DatasetQualityDetailsPublicStateUpdate = Partial< Pick< WithDefaultControllerState, - 'timeRange' | 'breakdownField' | 'expandedDegradedField' | 'showCurrentQualityIssues' + | 'timeRange' + | 'breakdownField' + | 'expandedDegradedField' + | 'showCurrentQualityIssues' + | 'qualityIssuesChart' > > & { dataStream: string; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts index edd16652374a1..dab87b672fb1f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts @@ -100,6 +100,8 @@ export const useDatasetQualityDetailsState = () => { rawName: dataStream, }; + const docsTrendChart = useSelector(service, (state) => state.context.qualityIssuesChart); + const loadingState = useSelector(service, (state) => ({ nonAggregatableDatasetLoading: state.matches('initializing.nonAggregatableDataset.fetching'), dataStreamDetailsLoading: state.matches('initializing.dataStreamDetails.fetching'), @@ -144,6 +146,7 @@ export const useDatasetQualityDetailsState = () => { datasetDetails, degradedFields, dataStreamDetails, + docsTrendChart, breakdownField, isBreakdownFieldEcs, isBreakdownFieldAsserted, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts index 328ffb2cebf42..70f4badcb4278 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_quality_issues_docs_chart.ts @@ -20,6 +20,7 @@ import { getLensAttributes as getFailedLensAttributes } from '../components/data import { useRedirectLink } from './use_redirect_link'; import { useDatasetDetailsTelemetry } from './use_dataset_details_telemetry'; import { useDatasetDetailsRedirectLinkTelemetry } from './use_redirect_link_telemetry'; +import { QualityIssue } from '../state_machines/dataset_quality_details_controller'; const exploreDataInLogsExplorerText = i18n.translate( 'xpack.datasetQuality.details.chartExploreDataInLogsExplorerText', @@ -43,9 +44,7 @@ const ACTION_EXPLORE_IN_LOGS_EXPLORER = 'ACTION_EXPLORE_IN_LOGS_EXPLORER'; const ACTION_OPEN_IN_LENS = 'ACTION_OPEN_IN_LENS'; const DEGRADED_DOCS_KUERY = `_ignored:*`; -export type QualityIssue = 'degradedDocs' | 'failedDocs'; - -export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { +export const useQualityIssuesDocsChart = () => { const { euiTheme } = useEuiTheme(); const { services: { lens }, @@ -55,6 +54,7 @@ export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { dataStream, datasetDetails, timeRange, + docsTrendChart, breakdownField, integrationDetails, isBreakdownFieldAsserted, @@ -72,7 +72,7 @@ export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { ReturnType | undefined >(undefined); - const query = qualityIssue === 'degradedDocs' ? DEGRADED_DOCS_KUERY : ''; + const query = docsTrendChart === 'degradedDocs' ? DEGRADED_DOCS_KUERY : ''; const { dataView } = useCreateDataView({ indexPatternString: getDataViewIndexPattern(dataStream), @@ -97,6 +97,16 @@ export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { [service] ); + const handleDocsTrendChartChange = useCallback( + (qualityIssuesChart: QualityIssue) => { + service.send({ + type: 'QUALITY_ISSUES_CHART_CHANGE', + qualityIssuesChart, + }); + }, + [service] + ); + useEffect(() => { if (isBreakdownFieldAsserted) trackDatasetDetailsBreakdownFieldChanged(); }, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]); @@ -108,7 +118,7 @@ export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; const lensAttributes = - qualityIssue === 'degradedDocs' + docsTrendChart === 'degradedDocs' ? getDegradedLensAttributes({ color: euiTheme.colors.danger, dataStream: dataStreamName, @@ -123,11 +133,11 @@ export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { }); setAttributes(lensAttributes); }, [ - qualityIssue, breakdownDataViewField?.name, euiTheme.colors.danger, setAttributes, dataStream, + docsTrendChart, integrationDetails?.integration?.datasets, datasetDetails.name, ]); @@ -227,6 +237,7 @@ export const useQualityIssuesDocsChart = (qualityIssue: QualityIssue) => { extraActions, isChartLoading, redirectLinkProps, + handleDocsTrendChartChange, onChartLoading: handleChartLoading, setAttributes, setIsChartLoading, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts index 26a51014b3abb..d54d5014090c4 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/defaults.ts @@ -30,4 +30,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = { refresh: DEFAULT_DATEPICKER_REFRESH, }, showCurrentQualityIssues: false, + qualityIssuesChart: 'degradedDocs', }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 8ac65a7dca4a7..d62f8be21398f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -121,6 +121,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( '#DatasetQualityDetailsController.initializing.checkBreakdownFieldIsEcs.fetching', actions: ['storeBreakDownField'], }, + QUALITY_ISSUES_CHART_CHANGE: { + target: 'done', + actions: ['storeQualityIssuesChart'], + }, }, }, }, @@ -450,6 +454,11 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( } : {}; }), + storeQualityIssuesChart: assign((_context, event) => { + return 'qualityIssuesChart' in event + ? { qualityIssuesChart: event.qualityIssuesChart } + : {}; + }), storeBreakDownField: assign((_context, event) => { return 'breakdownField' in event ? { breakdownField: event.breakdownField } : {}; }), diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index cdebcfbe53d86..eb5d4ea4f7429 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -22,6 +22,8 @@ import { import { TableCriteria, TimeRangeConfig } from '../../../common/types'; import { Integration } from '../../../common/data_streams_stats/integration'; +export type QualityIssue = 'degradedDocs' | 'failedDocs'; + export interface DataStream { name: string; type: string; @@ -50,6 +52,7 @@ export interface WithDefaultControllerState { degradedFields: DegradedFieldsTableConfig; timeRange: TimeRangeConfig; showCurrentQualityIssues: boolean; + qualityIssuesChart: QualityIssue; breakdownField?: string; isBreakdownFieldEcs?: boolean; isIndexNotFoundError?: boolean; @@ -108,7 +111,11 @@ export interface WithNewFieldLimitResponse { export type DefaultDatasetQualityDetailsContext = Pick< WithDefaultControllerState, - 'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues' + | 'degradedFields' + | 'timeRange' + | 'isIndexNotFoundError' + | 'showCurrentQualityIssues' + | 'qualityIssuesChart' >; export type DatasetQualityDetailsControllerTypeState = @@ -211,6 +218,10 @@ export type DatasetQualityDetailsControllerEvent = | { type: 'DEGRADED_FIELDS_LOADED'; } + | { + type: 'QUALITY_ISSUES_CHART_CHANGE'; + qualityIssuesChart: QualityIssue; + } | { type: 'BREAKDOWN_FIELD_CHANGE'; breakdownField: string | undefined; From 5ff14d6031e56cf16934b5a57bfc4a8959f1415b Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 19 Nov 2024 16:46:52 -0300 Subject: [PATCH 10/19] totalDocs should include also failedDocs --- .../hooks/use_dataset_details_telemetry.ts | 2 +- .../public/utils/generate_datasets.ts | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts index 2e88af3acf06d..77ccc5fd008cd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts @@ -169,7 +169,7 @@ function getDatasetDetailsEbtProps({ }; const degradedDocs = dataStreamDetails?.degradedDocsCount ?? 0; const failedDocs = dataStreamDetails?.failedDocsCount ?? 0; - const totalDocs = dataStreamDetails?.docsCount ?? 0; + const totalDocs = (dataStreamDetails?.docsCount ?? 0) + failedDocs; const degradedPercentage = calculatePercentage({ totalDocs, count: degradedDocs }); const failedPercentage = calculatePercentage({ totalDocs, count: failedDocs }); const health = mapPercentageToQuality([degradedPercentage, failedPercentage]); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index 3e170a493e7e1..d3723a02e67a3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -54,19 +54,19 @@ export function generateDatasets( const totalDocsMap: Record = Object.fromEntries(totalDocs.map(({ dataset, count }) => [dataset, count])); - const degradedMap: Record< + const failedMap: Record< DataStreamDocsStat['dataset'], { percentage: number; count: DataStreamDocsStat['count']; } - > = degradedDocStats.reduce( - (degradedMapAcc, { dataset, count }) => - Object.assign(degradedMapAcc, { + > = failedDocStats.reduce( + (failedMapAcc, { dataset, count }) => + Object.assign(failedMapAcc, { [dataset]: { count, percentage: calculatePercentage({ - totalDocs: totalDocsMap[dataset], + totalDocs: totalDocsMap[dataset] ? totalDocsMap[dataset] + count : 0, count, }), }, @@ -74,19 +74,19 @@ export function generateDatasets( {} ); - const failedMap: Record< + const degradedMap: Record< DataStreamDocsStat['dataset'], { percentage: number; count: DataStreamDocsStat['count']; } - > = failedDocStats.reduce( - (failedMapAcc, { dataset, count }) => - Object.assign(failedMapAcc, { + > = degradedDocStats.reduce( + (degradedMapAcc, { dataset, count }) => + Object.assign(degradedMapAcc, { [dataset]: { count, percentage: calculatePercentage({ - totalDocs: totalDocsMap[dataset] ? totalDocsMap[dataset] + count : 0, + totalDocs: totalDocsMap[dataset] + (failedMap[dataset]?.count ?? 0), count, }), }, From b70c9606885b35ccc56e92e623b60b34db2c1447 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 19 Nov 2024 17:05:27 -0300 Subject: [PATCH 11/19] fixing i18n problems --- x-pack/plugins/translations/translations/fr-FR.json | 1 - x-pack/plugins/translations/translations/ja-JP.json | 1 - x-pack/plugins/translations/translations/zh-CN.json | 1 - 3 files changed, 3 deletions(-) diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 6d1ba23a5f6c4..ef845b3e0b3e4 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -15364,7 +15364,6 @@ "xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "Nous n'avons pas pu obtenir d'informations sur les ensembles de données non agrégés.", "xpack.datasetQuality.fewDegradedDocsTooltip": "{degradedDocsCount} documents dégradés dans cet ensemble de données.", "xpack.datasetQuality.filterBar.placeholder": "Filtrer les ensembles de données", - "xpack.datasetQuality.flyout.degradedDocsTitle": "Documents dégradés", "xpack.datasetQuality.flyout.nonAggregatable.description": "{description}", "xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "{rolloverLink} manuellement cet ensemble de données pour empêcher des délais à l'avenir.", "xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset} est incompatible avec l'agrégation _ignored, ce qui peut entraîner des délais lors de la recherche de données. {howToFixIt}", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 04bc03184b777..aef21da9d5a3c 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -15343,7 +15343,6 @@ "xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "集約可能なデータセット情報以外を取得できませんでした。", "xpack.datasetQuality.fewDegradedDocsTooltip": "このデータセットの{degradedDocsCount}個の劣化したドキュメント。", "xpack.datasetQuality.filterBar.placeholder": "データセットのフィルタリング", - "xpack.datasetQuality.flyout.degradedDocsTitle": "劣化したドキュメント", "xpack.datasetQuality.flyout.nonAggregatable.description": "{description}", "xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "今後の遅れを防止するには、手動でこのデータを{rolloverLink}してください。", "xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset}は_ignored集約をサポートしていません。データのクエリを実行するときに遅延が生じる可能性があります。{howToFixIt}", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 1d48e398dfd0c..8bb46bd9edaa6 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -15037,7 +15037,6 @@ "xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "无法获取非可聚合数据集信息。", "xpack.datasetQuality.fewDegradedDocsTooltip": "此数据集中的 {degradedDocsCount} 个已降级文档。", "xpack.datasetQuality.filterBar.placeholder": "筛选数据集", - "xpack.datasetQuality.flyout.degradedDocsTitle": "已降级文档", "xpack.datasetQuality.flyout.nonAggregatable.description": "{description}", "xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "手动 {rolloverLink} 此数据集以防止未来出现延迟。", "xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset} 不支持 _ignored 聚合，在查询数据时可能会导致延迟。{howToFixIt}", From 36131aa61078b7e80ce523747b3e2d9aba563cbf Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 19 Nov 2024 17:35:53 -0300 Subject: [PATCH 12/19] fixing build --- .../table/failed_docs_percentage_link.tsx | 38 ++++++++++++++++--- .../dataset_quality/failed_docs.ts | 1 - 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx index 4f7db1dbbeb9d..2ba6b22b63771 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/failed_docs_percentage_link.tsx @@ -5,9 +5,10 @@ * 2.0. */ -import { EuiSkeletonRectangle, EuiFlexGroup } from '@elastic/eui'; +import { EuiSkeletonRectangle, EuiFlexGroup, EuiLink } from '@elastic/eui'; import React from 'react'; import { i18n } from '@kbn/i18n'; +import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks'; import { QualityPercentageIndicator } from '../../quality_indicator'; import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; import { TimeRangeConfig } from '../../../../common/types'; @@ -25,6 +26,18 @@ export const FailedDocsPercentageLink = ({ failedDocs: { percentage, count }, } = dataStreamStat; + const { sendTelemetry } = useDatasetRedirectLinkTelemetry({ + rawName: dataStreamStat.rawName, + query: { language: 'kuery', query: '' }, + }); + + const redirectLinkProps = useRedirectLink({ + dataStreamStat, + query: { language: 'kuery', query: '' }, + sendTelemetry, + timeRangeConfig: timeRange, + }); + const tooltip = (failedDocsCount: number) => i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', { defaultMessage: '{failedDocsCount} failed docs in this data set.', @@ -36,11 +49,24 @@ export const FailedDocsPercentageLink = ({ return ( - + {percentage ? ( + + + + ) : ( + + )} ); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts index 170da8def9439..1107644e499a2 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts @@ -27,7 +27,6 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const namespace = 'default'; const serviceName = 'my-service'; const hostName = 'synth-host'; - const dataStreamName = `${dataStreamType}-${dataset}-${namespace}`; const syntheticsDataStreamName = `${dataStreamType}-${syntheticsDataset}-${namespace}`; const endpoint = 'GET /internal/dataset_quality/data_streams/failed_docs'; From ad7d150ee0d812f52794ab9204f6c40aa31cdb9d Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Wed, 20 Nov 2024 15:03:58 -0300 Subject: [PATCH 13/19] fixing copies --- .../dataset_quality_details/overview/summary/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 13344ceb91767..617e5afc430c2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -26,7 +26,7 @@ import { DatasetQualityIndicator } from '../../../quality_indicator'; const degradedDocsTooltip = ( @@ -40,7 +40,7 @@ const degradedDocsTooltip = ( const failedDocsColumnTooltip = ( ); From 5f1b389faa6fc446c5be79daea1e2e54b0be014d Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Thu, 21 Nov 2024 15:03:38 -0300 Subject: [PATCH 14/19] Added e2e tests to fialure store support --- .../src/lib/logs/logs_synthtrace_es_client.ts | 24 +++++-- .../overview/document_trends/index.tsx | 1 + .../apps/dataset_quality/data/logs_data.ts | 65 +++++++++++++++++- .../dataset_quality_details.ts | 52 ++++++++++++++- .../dataset_quality/dataset_quality_table.ts | 35 ++++++++++ .../page_objects/dataset_quality.ts | 13 +++- .../dataset_quality_details.ts | 66 +++++++++++++++++-- .../dataset_quality/dataset_quality_table.ts | 37 ++++++++++- 8 files changed, 275 insertions(+), 18 deletions(-) diff --git a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts index 3e95383d9dbb9..6f1f1ca4ec6af 100644 --- a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts @@ -10,7 +10,11 @@ import { Client, estypes } from '@elastic/elasticsearch'; import { pipeline, Readable } from 'stream'; import { LogDocument } from '@kbn/apm-synthtrace-client/src/lib/logs'; -import { IngestProcessorContainer, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { + IndicesIndexSettings, + IngestProcessorContainer, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/types'; import { ValuesType } from 'utility-types'; import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; import { getSerializeTransform } from '../shared/get_serialize_transform'; @@ -52,7 +56,11 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { } } - async createComponentTemplate(name: string, mappings: MappingTypeMapping) { + async createComponentTemplate( + name: string, + mappings?: MappingTypeMapping, + settings?: IndicesIndexSettings + ) { const isTemplateExisting = await this.client.cluster.existsComponentTemplate({ name }); if (isTemplateExisting) return this.logger.info(`Component template already exists: ${name}`); @@ -61,7 +69,8 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { await this.client.cluster.putComponentTemplate({ name, template: { - mappings, + ...((mappings && { mappings }) || {}), + ...((settings && { settings }) || {}), }, }); this.logger.info(`Component template successfully created: ${name}`); @@ -124,16 +133,17 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { } } - async createCustomPipeline(processors: IngestProcessorContainer[]) { + async createCustomPipeline(processors: IngestProcessorContainer[], pipelineId?: string) { + const id = pipelineId ?? LogsCustom; try { this.client.ingest.putPipeline({ - id: LogsCustom, + id, processors, version: 1, }); - this.logger.info(`Custom pipeline created: ${LogsCustom}`); + this.logger.info(`Custom pipeline created: ${id}`); } catch (err) { - this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`); + this.logger.error(`Custom pipeline creation failed: ${id} - ${err.message}`); } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx index 541a6bee711a0..0206de6606afc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx @@ -97,6 +97,7 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num { + return Array(count) + .fill(0) + .flatMap((_, index) => { + const isFailed = index % (count * rate) === 0; + return log + .create() + .dataset(dataset) + .message(MESSAGE_LOG_LEVELS[0].message) + .logLevel(isFailed ? 'anyLevel' : LogLevel.INFO) + .service(SERVICE_NAMES[0]) + .namespace(namespace ?? defaultNamespace) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + }) + .timestamp(timestamp); + }); + }); +} + +export const customLogLevelProcessor = [ + { + script: { + tag: 'normalize log level', + lang: 'painless', + source: ` + String level = ctx['log.level']; + if ('info'.equals(level)) { + ctx['log.level'] = 'info'; + } else if ('debug'.equals(level)) { + ctx['log.level'] = 'debug'; + } else if ('error'.equals(level)) { + ctx['log.level'] = 'error'; + } else { + throw new Exception("Not a valid log level"); + } + `, + }, + }, +]; + export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; export const defaultNamespace = 'default'; export const productionNamespace = 'production'; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts index 3b93f0ceccfeb..06659eb27e4de 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts @@ -6,9 +6,12 @@ */ import expect from '@kbn/expect'; +import merge from 'lodash/merge'; import { DatasetQualityFtrProviderContext } from './config'; import { createDegradedFieldsRecord, + createFailedRecords, + customLogLevelProcessor, datasetNames, defaultNamespace, getInitialTestLogs, @@ -54,6 +57,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const regularDataStreamName = `logs-${datasetNames[0]}-${defaultNamespace}`; const degradedDatasetName = datasetNames[2]; const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const failedDatasetName = datasetNames[1]; + const failedDataStreamName = `logs-${failedDatasetName}-${defaultNamespace}`; describe('Dataset Quality Details', () => { before(async () => { @@ -63,6 +68,25 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Install Bitbucket Integration (package which does not has Dashboards) and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(bitbucketPkg); + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor, 'logs-apache.access@custom'); + await synthtrace.createComponentTemplate('logs-apache.access@custom', undefined, { + 'index.default_pipeline': 'logs-apache.access@custom', + }); + await synthtrace.updateIndexTemplate( + 'logs-apache.access', + (template: Record): Record => { + const next: Record = { + name: 'logs-apache.access', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + await synthtrace.index([ // Ingest basic logs getInitialTestLogs({ to, count: 4 }), @@ -87,6 +111,14 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid namespace: productionNamespace, isMalformed: true, }), + // Index failed docs for Apache integration + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: apacheAccessDatasetName, + namespace: productionNamespace, + rate: 0.5, + }), // Index logs for Bitbucket integration getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName }), ]); @@ -160,6 +192,19 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expect(currentUrl).to.not.contain('breakdownField'); }); }); + + it('reflects the selected quality issue chart state in url', async () => { + await PageObjects.datasetQuality.navigateToDetails({ dataStream: failedDataStreamName }); + + const charType = 'failedDocs'; + await PageObjects.datasetQuality.selectQualityIssuesChartType(charType); + + // Wait for URL to contain "qualityIssuesChart:failedDocs" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(decodeURIComponent(currentUrl)).to.contain(`qualityIssuesChart:${charType}`); + }); + }); }); describe('overview summary panel', () => { @@ -168,13 +213,14 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid dataStream: apacheAccessDataStreamName, }); - const { docsCountTotal, degradedDocs, services, hosts, size } = + const { docsCountTotal, degradedDocs, failedDocs, services, hosts, size } = await PageObjects.datasetQuality.parseOverviewSummaryPanelKpis(); - expect(parseInt(docsCountTotal, 10)).to.be(226); + expect(parseInt(docsCountTotal, 10)).to.be(306); expect(parseInt(degradedDocs, 10)).to.be(1); expect(parseInt(services, 10)).to.be(3); expect(parseInt(hosts, 10)).to.be(52); expect(parseInt(size, 10)).to.be.greaterThan(0); + expect(parseInt(failedDocs, 10)).to.be(20); }); }); @@ -371,7 +417,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const rows = await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - expect(rows.length).to.eql(3); + expect(rows.length).to.eql(2); }); it('should display Spark Plot for every row of degraded fields', async () => { diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index 233a7b5e6b3ba..bb9c4699003ad 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -6,8 +6,11 @@ */ import expect from '@kbn/expect'; +import merge from 'lodash/merge'; import { DatasetQualityFtrProviderContext } from './config'; import { + createFailedRecords, + customLogLevelProcessor, datasetNames, defaultNamespace, getInitialTestLogs, @@ -26,6 +29,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const to = '2024-01-01T12:00:00.000Z'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; + const failedDatasetName = 'synth.failed'; const pkg = { name: 'apache', version: '1.14.0', @@ -33,6 +37,22 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid describe('Dataset quality table', () => { before(async () => { + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor); + await synthtrace.updateIndexTemplate( + 'logs', + (template: Record): Record => { + const next: Record = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + // Install Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(pkg); // Ingest basic logs @@ -53,6 +73,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.datasetQuality.navigateTo(); }); @@ -109,6 +136,14 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); }); + it('shows failed docs percentage', async () => { + const cols = await PageObjects.datasetQuality.parseDatasetTable(); + + const failedDocsCol = cols['Failed docs (%)']; + const failedDocsColCellTexts = await failedDocsCol.getCellTexts(); + expect(failedDocsColCellTexts).to.eql(['0%', '0%', '0%', '0%', '20%']); + }); + it('shows the value in the size column', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index cef881fe0797c..4d10e0d04cf96 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -54,7 +54,7 @@ type SummaryPanelKpi = Record< >; type SummaryPanelKPI = Record< - 'docsCountTotal' | 'size' | 'services' | 'hosts' | 'degradedDocs', + 'docsCountTotal' | 'size' | 'services' | 'hosts' | 'degradedDocs' | 'failedDocs', string >; @@ -70,6 +70,7 @@ const texts = { services: 'Services', hosts: 'Hosts', degradedDocs: 'Degraded docs', + failedDocs: 'Failed docs', }; export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProviderContext) { @@ -138,6 +139,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist', datasetQualityDetailsOverviewDegradedFieldToggleSwitch: 'datasetQualityDetailsOverviewDegradedFieldToggleSwitch', + datasetQualityIssuesChartTypeButtonGroup: 'datasetQualityDetailsChartTypeButtonGroup', }; return { @@ -395,6 +397,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv { title: texts.services, key: 'services' }, { title: texts.hosts, key: 'hosts' }, { title: texts.degradedDocs, key: 'degradedDocs' }, + { title: texts.failedDocs, key: 'failedDocs' }, ].filter((item) => !excludeKeys.includes(item.key)); const kpiTexts = await Promise.all( @@ -415,6 +418,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ); }, + async selectQualityIssuesChartType(chartType: 'degradedDocs' | 'failedDocs') { + const datasetDetailsContainer: WebElementWrapper = await testSubjects.find( + testSubjectSelectors.datasetQualityIssuesChartTypeButtonGroup + ); + const refreshButton = await datasetDetailsContainer.findByTestSubject(chartType); + return refreshButton.click(); + }, + /** * Selects a breakdown field from the unified histogram breakdown selector * @param fieldText The text of the field to select. Use 'No breakdown' to clear the selection diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts index 0d8d8e8865d52..1f0abda2242a3 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts @@ -6,7 +6,16 @@ */ import expect from '@kbn/expect'; -import { defaultNamespace } from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; +import { + createFailedRecords, + customLogLevelProcessor, + defaultNamespace, +} from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; +import merge from 'lodash/merge'; +import { + IndicesIndexTemplate, + IndicesPutIndexTemplateRequest, +} from '@elastic/elasticsearch/lib/api/types'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { datasetNames, @@ -55,6 +64,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const regularDataStreamName = `logs-${datasetNames[0]}-${defaultNamespace}`; const degradedDatasetName = datasetNames[2]; const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`; + const failedDatasetName = datasetNames[1]; + const failedDataStreamName = `logs-${failedDatasetName}-${defaultNamespace}`; describe('Dataset quality details', function () { before(async () => { @@ -64,6 +75,31 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Install Bitbucket Integration (package which does not has Dashboards) and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(bitbucketPkg); + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor, 'logs-apache.access@custom'); + await synthtrace.createComponentTemplate('logs-apache.access@custom', undefined, { + 'index.default_pipeline': 'logs-apache.access@custom', + }); + await synthtrace.updateIndexTemplate( + 'logs-apache.access', + (template: IndicesIndexTemplate): IndicesPutIndexTemplateRequest => { + const next = { + name: 'logs-apache.access', + index_patterns: template.index_patterns, + template: { + settings: template.template?.settings, + mappings: template.template?.mappings, + aliases: template.template?.aliases, + }, + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + } + ); + await synthtrace.index([ // Ingest basic logs getInitialTestLogs({ to, count: 4 }), @@ -88,6 +124,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { namespace: productionNamespace, isMalformed: true, }), + // Index failed docs for Apache integration + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: apacheAccessDatasetName, + namespace: productionNamespace, + rate: 0.5, + }), // Index logs for Bitbucket integration getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName }), ]); @@ -163,6 +207,19 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(currentUrl).to.not.contain('breakdownField'); }); }); + + it('reflects the selected quality issue chart state in url', async () => { + await PageObjects.datasetQuality.navigateToDetails({ dataStream: failedDataStreamName }); + + const charType = 'failedDocs'; + await PageObjects.datasetQuality.selectQualityIssuesChartType(charType); + + // Wait for URL to contain "qualityIssuesChart:failedDocs" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(decodeURIComponent(currentUrl)).to.contain(`qualityIssuesChart:${charType}`); + }); + }); }); // FLAKY: https://github.com/elastic/kibana/issues/194575 @@ -172,15 +229,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataStream: apacheAccessDataStreamName, }); - const { docsCountTotal, degradedDocs, services, hosts, size } = + const { docsCountTotal, degradedDocs, failedDocs, services, hosts, size } = await PageObjects.datasetQuality.parseOverviewSummaryPanelKpis(); - expect(parseInt(docsCountTotal, 10)).to.be(226); + expect(parseInt(docsCountTotal, 10)).to.be(306); expect(parseInt(degradedDocs, 10)).to.be(1); expect(parseInt(services, 10)).to.be(3); expect(parseInt(hosts, 10)).to.be(52); // metering stats API is cached for 30seconds, waiting for the exact value is not optimal in this case // rather we can just check if any value is present expect(size).to.be.ok(); + expect(parseInt(failedDocs, 10)).to.be(20); }); }); @@ -377,7 +435,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const rows = await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); - expect(rows.length).to.eql(3); + expect(rows.length).to.eql(2); }); it('should display Spark Plot for every row of degraded fields', async () => { diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts index c42b50116ea95..1896016448538 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts @@ -6,7 +6,12 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; +import { + createFailedRecords, + customLogLevelProcessor, +} from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; +import merge from 'lodash/merge'; +import { IndicesIndexTemplate } from '@elastic/elasticsearch/lib/api/types'; import { datasetNames, defaultNamespace, @@ -14,6 +19,7 @@ import { getLogsForDataset, productionNamespace, } from './data'; +import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects([ @@ -28,6 +34,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const to = '2024-01-01T12:00:00.000Z'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; + const failedDatasetName = 'synth.failed'; const pkg = { name: 'apache', version: '1.14.0', @@ -35,6 +42,19 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Dataset quality table', function () { before(async () => { + // Enable failure store for logs + await synthtrace.createCustomPipeline(customLogLevelProcessor); + await synthtrace.updateIndexTemplate('logs', (template: IndicesIndexTemplate) => { + const next = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + }); + // Install Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(pkg); // Ingest basic logs @@ -55,6 +75,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.svlCommonPage.loginAsAdmin(); await PageObjects.datasetQuality.navigateTo(); @@ -112,6 +139,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); }); + it('shows failed docs percentage', async () => { + const cols = await PageObjects.datasetQuality.parseDatasetTable(); + + const failedDocsCol = cols['Failed docs (%)']; + const failedDocsColCellTexts = await failedDocsCol.getCellTexts(); + expect(failedDocsColCellTexts).to.eql(['0%', '0%', '0%', '0%', '20%']); + }); + it('shows the value in the size column', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); From 3715579c069dd78607b696632502f1608b1ad1d0 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Thu, 21 Nov 2024 15:29:36 -0300 Subject: [PATCH 15/19] fixing i18n id duplicated --- .../dataset_quality_details/overview/summary/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 617e5afc430c2..a1279895b24db 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -39,7 +39,7 @@ const degradedDocsTooltip = ( const failedDocsColumnTooltip = ( ); From dd9816efd5940d0933e1a92e93d2445e1ccb02b2 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Fri, 22 Nov 2024 13:17:43 -0300 Subject: [PATCH 16/19] fixing some e2e tests --- .../public/utils/generate_datasets.test.ts | 2 +- .../test/functional/page_objects/dataset_quality.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index c910c0b15fc96..2b4a25d60313d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -117,7 +117,7 @@ describe('generateDatasets', () => { count: 0, }, failedDocs: { - percentage: 1.9607843137254901, + percentage: 1.96, count: 2, }, }, diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index 4d10e0d04cf96..389342d0b3ecf 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -301,13 +301,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv await this.waitUntilTableLoaded(); const table = await this.getDatasetsTable(); return this.parseTable(table, [ - '0', - 'Data Set Name', + 'Data set name', 'Namespace', + 'Type', 'Size', - 'Data Set Quality', - 'Degraded Docs (%)', - 'Last Activity', + 'Data set quality', + 'Degraded docs (%)', + 'Failed docs (%)', + 'Last activity', 'Actions', ]); }, From 076cb6c5589cb5f96a45187de7b0ebd2fd2b4c6f Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 25 Nov 2024 09:58:51 -0300 Subject: [PATCH 17/19] fixing e2e tests --- .../public/utils/generate_datasets.ts | 9 +++--- .../dataset_quality/failed_docs.ts | 31 ++++++++++--------- .../apps/dataset_quality/data/logs_data.ts | 2 +- .../dataset_quality/dataset_quality_table.ts | 9 ++++-- .../dataset_quality/data/logs_data.ts | 2 +- .../dataset_quality/dataset_quality_table.ts | 11 ++++--- 6 files changed, 37 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index d3723a02e67a3..82eb41f5b1651 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -66,7 +66,7 @@ export function generateDatasets( [dataset]: { count, percentage: calculatePercentage({ - totalDocs: totalDocsMap[dataset] ? totalDocsMap[dataset] + count : 0, + totalDocs: (totalDocsMap[dataset] ?? 0) + count, count, }), }, @@ -86,7 +86,7 @@ export function generateDatasets( [dataset]: { count, percentage: calculatePercentage({ - totalDocs: totalDocsMap[dataset] + (failedMap[dataset]?.count ?? 0), + totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0), count, }), }, @@ -109,7 +109,7 @@ export function generateDatasets( degradedDocStat: degradedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, failedDocStat: failedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, datasetIntegrationMap, - totalDocs: totalDocsMap[dataset] ?? 0, + totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0), }) ); } @@ -129,7 +129,8 @@ export function generateDatasets( integrationsMap[dataStream.integration ?? ''], degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs, failedDocs: failedMap[dataset.rawName] || dataset.failedDocs, - docsInTimeRange: totalDocsMap[dataset.rawName] ?? 0, + docsInTimeRange: + (totalDocsMap[dataset.rawName] ?? 0) + (failedMap[dataset.rawName]?.count ?? 0), quality: mapPercentageToQuality(qualityStats), }; }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts index 1107644e499a2..1c711b447876b 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts @@ -19,6 +19,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const samlAuth = getService('samlAuth'); const roleScopedSupertest = getService('roleScopedSupertest'); const synthtrace = getService('synthtrace'); + const retry = getService('retry'); const from = '2024-09-20T11:00:00.000Z'; const to = '2024-09-20T11:01:00.000Z'; const dataStreamType = 'logs'; @@ -108,9 +109,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { .create() .message('This is a log message') .timestamp(timestamp) - .dataset(dataset) + .dataset(syntheticsDataset) .namespace(namespace) - .logLevel('0') + .logLevel('5') .defaults({ 'log.file.path': '/my-service.log', 'service.name': serviceName, @@ -120,9 +121,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { .create() .message('This is a log message') .timestamp(timestamp) - .dataset(syntheticsDataset) + .dataset(dataset) .namespace(namespace) - .logLevel('5') + .logLevel('0') .defaults({ 'log.file.path': '/my-service.log', 'service.name': serviceName, @@ -138,17 +139,19 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }); it('returns number of failed documents per DataStream', async () => { - const resp = await callApiAs({ - roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, - apiParams: { - start: from, - end: to, - }, - }); + await retry.tryForTime(180 * 1000, async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: from, + end: new Date().toISOString(), + }, + }); - expect(resp.body.failedDocs.length).to.be(1); - expect(resp.body.totalDocs[0].dataset).to.be(syntheticsDataStreamName); - expect(resp.body.totalDocs[0].count).to.be(1); + expect(resp.body.failedDocs.length).to.be(1); + expect(resp.body.failedDocs[0].dataset).to.be(syntheticsDataStreamName); + expect(resp.body.failedDocs[0].count).to.be(1); + }); }); it('returns empty when all documents are outside timeRange', async () => { diff --git a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts index f28e1c41f4541..17bb7aed54c81 100644 --- a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts +++ b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts @@ -250,7 +250,7 @@ export const customLogLevelProcessor = [ }, ]; -export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; +export const datasetNames = ['synth.1', 'synth.2', 'synth.3', 'synth.failed']; export const defaultNamespace = 'default'; export const productionNamespace = 'production'; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index bb9c4699003ad..4edacd68f02a0 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -104,6 +104,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid defaultNamespace, defaultNamespace, defaultNamespace, + defaultNamespace, productionNamespace, ]); @@ -115,11 +116,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const cols = await PageObjects.datasetQuality.parseDatasetTable(); const lastActivityCol = cols['Last activity']; const activityCells = await lastActivityCol.getCellTexts(); - const lastActivityCell = activityCells[activityCells.length - 1]; - const restActivityCells = activityCells.slice(0, -1); + const lastActivityDegradedCell = activityCells[activityCells.length - 2]; + const lastActivityFailedCell = activityCells[activityCells.length - 1]; + const restActivityCells = activityCells.slice(0, -2); // The first cell of lastActivity should have data - expect(lastActivityCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityDegradedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityFailedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); // The rest of the rows must show no activity expect(restActivityCells).to.eql([ PageObjects.datasetQuality.texts.noActivityText, diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts index 3692a17709a2e..338177a7b6b9e 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts @@ -189,7 +189,7 @@ export function createDegradedFieldsRecord({ }); } -export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; +export const datasetNames = ['synth.1', 'synth.2', 'synth.3', 'synth.failed']; export const defaultNamespace = 'default'; export const productionNamespace = 'production'; diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts index 1896016448538..a8e8cf9bf6b7b 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts @@ -107,6 +107,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { defaultNamespace, defaultNamespace, defaultNamespace, + defaultNamespace, productionNamespace, ]); @@ -118,11 +119,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const cols = await PageObjects.datasetQuality.parseDatasetTable(); const lastActivityCol = cols['Last activity']; const activityCells = await lastActivityCol.getCellTexts(); - const lastActivityCell = activityCells[activityCells.length - 1]; - const restActivityCells = activityCells.slice(0, -1); + const lastActivityDegradedCell = activityCells[activityCells.length - 2]; + const lastActivityFailedCell = activityCells[activityCells.length - 1]; + const restActivityCells = activityCells.slice(0, -2); // The first cell of lastActivity should have data - expect(lastActivityCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityDegradedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); + expect(lastActivityFailedCell).to.not.eql(PageObjects.datasetQuality.texts.noActivityText); // The rest of the rows must show no activity expect(restActivityCells).to.eql([ PageObjects.datasetQuality.texts.noActivityText, @@ -136,7 +139,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const degradedDocsCol = cols['Degraded docs (%)']; const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts(); - expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); + expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%', '0%']); }); it('shows failed docs percentage', async () => { From f10abac04bc3fd8321ed0276cb26794643c2b73e Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 26 Nov 2024 15:36:16 -0300 Subject: [PATCH 18/19] fixing tests --- .../public/utils/generate_datasets.test.ts | 8 ++++---- .../apps/dataset_quality/dataset_quality_table.ts | 2 +- .../dataset_quality/dataset_quality_table_filters.ts | 10 +++++----- .../dataset_quality/dataset_quality_table_filters.ts | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index 2b4a25d60313d..135979e63fb60 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -110,7 +110,7 @@ describe('generateDatasets', () => { userPrivileges: { canMonitor: true, }, - docsInTimeRange: 100, + docsInTimeRange: 102, quality: 'degraded', degradedDocs: { percentage: 0, @@ -173,14 +173,14 @@ describe('generateDatasets', () => { userPrivileges: { canMonitor: true, }, - docsInTimeRange: 0, - quality: 'good', + docsInTimeRange: 2, + quality: 'poor', degradedDocs: { percentage: 0, count: 0, }, failedDocs: { - percentage: 0, + percentage: 100, count: 2, }, }, diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index 4edacd68f02a0..32a3368d435a2 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -136,7 +136,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const degradedDocsCol = cols['Degraded docs (%)']; const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts(); - expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']); + expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%', '0%']); }); it('shows failed docs percentage', async () => { diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts index 3c8c3e702d576..8f485af363e48 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts @@ -61,7 +61,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('shows full dataset names when toggled', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -83,7 +83,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('searches the datasets', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -94,7 +94,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ); const colsAfterSearch = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterSearch = colsAfterSearch['Data Set Name']; + const datasetNameColAfterSearch = colsAfterSearch['Data set name']; const datasetNameColCellTextsAfterSearch = await datasetNameColAfterSearch.getCellTexts(); expect(datasetNameColCellTextsAfterSearch).to.eql([datasetNames[2]]); @@ -104,7 +104,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid it('filters for integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -112,7 +112,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.filterForIntegrations([apacheIntegrationName]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterFilter = colsAfterFilter['Data Set Name']; + const datasetNameColAfterFilter = colsAfterFilter['Data set name']; const datasetNameColCellTextsAfterFilter = await datasetNameColAfterFilter.getCellTexts(); expect(datasetNameColCellTextsAfterFilter).to.eql([apacheAccessDatasetHumanName]); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts index d57f852c0e700..6a0742367584d 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts @@ -63,7 +63,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows full dataset names when toggled', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -84,7 +84,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('searches the datasets', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -95,7 +95,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); const colsAfterSearch = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterSearch = colsAfterSearch['Data Set Name']; + const datasetNameColAfterSearch = colsAfterSearch['Data set name']; const datasetNameColCellTextsAfterSearch = await datasetNameColAfterSearch.getCellTexts(); expect(datasetNameColCellTextsAfterSearch).to.eql([datasetNames[2]]); // Reset the search field @@ -104,7 +104,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('filters for integration', async () => { const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Data Set Name']; + const datasetNameCol = cols['Data set name']; const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); expect(datasetNameColCellTexts).to.eql(allDatasetNames); @@ -112,7 +112,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.filterForIntegrations([apacheIntegrationName]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameColAfterFilter = colsAfterFilter['Data Set Name']; + const datasetNameColAfterFilter = colsAfterFilter['Data set name']; const datasetNameColCellTextsAfterFilter = await datasetNameColAfterFilter.getCellTexts(); expect(datasetNameColCellTextsAfterFilter).to.eql([apacheAccessDatasetHumanName]); // Reset the filter by selecting from the dropdown again From f9ba989698cdc5f3b65541a69aba20562b941a63 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Wed, 27 Nov 2024 10:04:08 -0300 Subject: [PATCH 19/19] fixing some tests --- .../dataset_quality_table_filters.ts | 20 ++++++++++++++++--- .../dataset_quality_table_filters.ts | 13 ++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts index 8f485af363e48..51b4163fa71fe 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table_filters.ts @@ -7,7 +7,13 @@ import expect from '@kbn/expect'; import { DatasetQualityFtrProviderContext } from './config'; -import { datasetNames, getInitialTestLogs, getLogsForDataset, productionNamespace } from './data'; +import { + createFailedRecords, + datasetNames, + getInitialTestLogs, + getLogsForDataset, + productionNamespace, +} from './data'; export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { const PageObjects = getPageObjects([ @@ -19,6 +25,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const synthtrace = getService('logSynthtraceEsClient'); const testSubjects = getService('testSubjects'); const to = '2024-01-01T12:00:00.000Z'; + const failedDatasetName = 'synth.failed'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; const apacheIntegrationName = 'Apache HTTP Server'; @@ -50,6 +57,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.datasetQuality.navigateTo(); }); @@ -143,7 +157,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const expectedQuality = 'Poor'; // Get default quality const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQuality = cols['Data Set Quality']; + const datasetQuality = cols['Data set quality']; const datasetQualityCellTexts = await datasetQuality.getCellTexts(); expect(datasetQualityCellTexts).to.contain(expectedQuality); @@ -151,7 +165,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.filterForQualities([expectedQuality]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQualityAfterFilter = colsAfterFilter['Data Set Quality']; + const datasetQualityAfterFilter = colsAfterFilter['Data set quality']; const datasetQualityCellTextsAfterFilter = await datasetQualityAfterFilter.getCellTexts(); expect(datasetQualityCellTextsAfterFilter).to.eql([expectedQuality]); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts index 6a0742367584d..972b7bb420162 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table_filters.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { createFailedRecords } from '@kbn/test-suites-xpack/functional/apps/dataset_quality/data'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { datasetNames, getInitialTestLogs, getLogsForDataset, productionNamespace } from './data'; @@ -20,6 +21,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const synthtrace = getService('svlLogsSynthtraceClient'); const testSubjects = getService('testSubjects'); const to = '2024-01-01T12:00:00.000Z'; + const failedDatasetName = 'synth.failed'; const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; const apacheIntegrationName = 'Apache HTTP Server'; @@ -51,6 +53,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { dataset: apacheAccessDatasetName, namespace: productionNamespace, }), + // Ingest Failed Logs + createFailedRecords({ + to: new Date().toISOString(), + count: 10, + dataset: failedDatasetName, + rate: 0.5, + }), ]); await PageObjects.svlCommonPage.loginWithPrivilegedRole(); await PageObjects.datasetQuality.navigateTo(); @@ -142,7 +151,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const expectedQuality = 'Poor'; // Get default quality const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQuality = cols['Data Set Quality']; + const datasetQuality = cols['Data set quality']; const datasetQualityCellTexts = await datasetQuality.getCellTexts(); expect(datasetQualityCellTexts).to.contain(expectedQuality); @@ -150,7 +159,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.filterForQualities([expectedQuality]); const colsAfterFilter = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetQualityAfterFilter = colsAfterFilter['Data Set Quality']; + const datasetQualityAfterFilter = colsAfterFilter['Data set quality']; const datasetQualityCellTextsAfterFilter = await datasetQualityAfterFilter.getCellTexts(); expect(datasetQualityCellTextsAfterFilter).to.eql([expectedQuality]);