diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index a293112eed548..1ddd0bdbcc9c0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -188,6 +188,7 @@ export type DataStreamSettings = rt.TypeOf; export const dataStreamDetailsRt = rt.partial({ lastActivity: rt.number, degradedDocsCount: rt.number, + failedDocsCount: rt.number, docsCount: rt.number, sizeBytes: rt.number, services: rt.record(rt.string, rt.array(rt.string)), diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 53f52975d8d98..5709687726267 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -93,12 +93,9 @@ export const flyoutSummaryText = i18n.translate('xpack.datasetQuality.flyoutSumm defaultMessage: 'Summary', }); -export const overviewDegradedDocsText = i18n.translate( - 'xpack.datasetQuality.flyout.degradedDocsTitle', - { - defaultMessage: 'Degraded docs', - } -); +export const overviewTrendsDocsText = i18n.translate('xpack.datasetQuality.flyout.trendDocsTitle', { + defaultMessage: 'Document trends', +}); export const flyoutDegradedDocsTrendText = i18n.translate( 'xpack.datasetQuality.flyoutDegradedDocsViz', @@ -107,6 +104,13 @@ export const flyoutDegradedDocsTrendText = i18n.translate( } ); +export const flyoutFailedDocsTrendText = i18n.translate( + 'xpack.datasetQuality.flyoutFailedDocsViz', + { + defaultMessage: 'Failed documents trend', + } +); + export const flyoutDegradedDocsPercentageText = i18n.translate( 'xpack.datasetQuality.flyoutDegradedDocsPercentage', { @@ -115,6 +119,14 @@ export const flyoutDegradedDocsPercentageText = i18n.translate( } ); +export const flyoutFailedDocsPercentageText = i18n.translate( + 'xpack.datasetQuality.flyoutFailedDocsPercentage', + { + defaultMessage: 'Failed docs %', + description: 'Tooltip label for the percentage of failed documents chart.', + } +); + export const flyoutDocsCountTotalText = i18n.translate( 'xpack.datasetQuality.flyoutDocsCountTotal', { @@ -319,6 +331,13 @@ export const overviewPanelDatasetQualityIndicatorDegradedDocs = i18n.translate( } ); +export const overviewPanelDatasetQualityIndicatorFailedDocs = i18n.translate( + 'xpack.datasetQuality.details.overviewPanel.datasetQuality.failedDocs', + { + defaultMessage: 'Failed docs', + } +); + export const overviewDegradedFieldsTableLoadingText = i18n.translate( 'xpack.datasetQuality.details.degradedFieldsTableLoadingText', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx index 05de567a6dab7..347c970577446 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/degraded_docs/index.tsx @@ -9,8 +9,8 @@ import React, { useCallback, useEffect, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiAccordion, + EuiButtonGroup, EuiButtonIcon, - EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, @@ -25,12 +25,13 @@ import { import type { DataViewField } from '@kbn/data-views-plugin/common'; import { css } from '@emotion/react'; import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; +import { useFailedDocsChart } from '../../../../../hooks/use_failed_docs_chart'; import { discoverAriaText, logsExplorerAriaText, openInDiscoverText, openInLogsExplorerText, - overviewDegradedDocsText, + overviewTrendsDocsText, } from '../../../../../../common/translations'; import { DegradedDocsChart } from './degraded_docs_chart'; import { @@ -42,53 +43,62 @@ import { import { _IGNORED } from '../../../../../../common/es_fields'; import { NavigationSource } from '../../../../../services/telemetry'; -const degradedDocsTooltip = ( +const trendDocsTooltip = ( - _ignored - - ), - }} + id="xpack.datasetQuality.details.trendDocsTooltip" + defaultMessage="The percentage of ignored fields or failed docs over the selected timeframe." /> ); +const DEGRADED_DOCS_KUERY = `${_IGNORED}: *`; + // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: number }) { const { timeRange, updateTimeRange, datasetDetails } = useDatasetQualityDetailsState(); - const { dataView, breakdown, ...chartProps } = useDegradedDocsChart(); + const { + dataView: degradedDataView, + breakdown: degradedBreakdown, + ...degradeChartProps + } = useDegradedDocsChart(); + const { + dataView: failedDataView, + breakdown: failedBreakDown, + ...failedChartProps + } = useFailedDocsChart(); const accordionId = useGeneratedHtmlId({ - prefix: overviewDegradedDocsText, + prefix: overviewTrendsDocsText, }); const [breakdownDataViewField, setBreakdownDataViewField] = useState( undefined ); + const [selectedChart, setSelectedChart] = useState('degradedDocs'); + const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ - query: { language: 'kuery', query: `${_IGNORED}: *` }, + query: { language: 'kuery', query: DEGRADED_DOCS_KUERY }, navigationSource: NavigationSource.Trend, }); const degradedDocLinkLogsExplorer = useRedirectLink({ dataStreamStat: datasetDetails, timeRangeConfig: timeRange, - query: { language: 'kuery', query: `${_IGNORED}: *` }, + query: { + language: 'kuery', + query: selectedChart === 'degradedDocs' ? DEGRADED_DOCS_KUERY : '', + }, sendTelemetry, }); useEffect(() => { - if (breakdown.dataViewField && breakdown.fieldSupportsBreakdown) { - setBreakdownDataViewField(breakdown.dataViewField); + if (degradedBreakdown.dataViewField && degradedBreakdown.fieldSupportsBreakdown) { + setBreakdownDataViewField(degradedBreakdown.dataViewField); } else { setBreakdownDataViewField(undefined); } - }, [setBreakdownDataViewField, breakdown]); + }, [setBreakdownDataViewField, degradedBreakdown]); const onTimeRangeChange = useCallback( ({ start, end }: Pick) => { @@ -107,9 +117,9 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe `} > -
{overviewDegradedDocsText}
+
{overviewTrendsDocsText}
- + @@ -124,14 +134,30 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe initialIsOpen={true} data-test-subj="datasetQualityDetailsOverviewDocumentTrends" > + - - + - + + {selectedChart === 'degradedDocs' ? ( + + + + ) : ( + <> + )} - - - + {selectedChart === 'degradedDocs' ? ( + + ) : ( + + )} ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts new file mode 100644 index 0000000000000..8f4b9f4bfcd4d --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes.ts @@ -0,0 +1,276 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public'; +import { v4 as uuidv4 } from 'uuid'; + +import { + flyoutFailedDocsTrendText, + flyoutFailedDocsPercentageText, +} from '../../../../../../common/translations'; + +enum DatasetQualityLensColumn { + Date = 'date_column', + FailedDocs = 'failed_docs_column', + CountFailed = 'count_failed', + CountTotal = 'count_total', + Math = 'math_column', + Breakdown = 'breakdown_column', +} + +const MAX_BREAKDOWN_SERIES = 5; + +interface GetLensAttributesParams { + color: string; + dataStream: string; + datasetTitle: string; + breakdownFieldName?: string; +} + +export function getLensAttributes({ + color, + dataStream, + datasetTitle, + breakdownFieldName, +}: GetLensAttributesParams) { + const dataViewId = uuidv4(); + + const columnOrder = [ + DatasetQualityLensColumn.Date, + DatasetQualityLensColumn.CountFailed, + DatasetQualityLensColumn.CountTotal, + DatasetQualityLensColumn.Math, + DatasetQualityLensColumn.FailedDocs, + ]; + + if (breakdownFieldName) { + columnOrder.unshift(DatasetQualityLensColumn.Breakdown); + } + + const columns = getChartColumns(breakdownFieldName); + return { + visualizationType: 'lnsXY', + title: flyoutFailedDocsTrendText, + references: [], + state: { + ...getAdHocDataViewState(dataViewId, dataStream, datasetTitle), + datasourceStates: { + formBased: { + layers: { + layer1: { + columnOrder, + columns, + indexPatternId: dataViewId, + }, + }, + }, + }, + filters: [], + query: { + language: 'kuery', + query: '', + }, + visualization: { + axisTitlesVisibilitySettings: { + x: false, + yLeft: false, + yRight: false, + }, + fittingFunction: 'None', + gridlinesVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + layers: [ + { + accessors: [DatasetQualityLensColumn.FailedDocs], + layerId: 'layer1', + layerType: 'data', + seriesType: 'line', + xAccessor: DatasetQualityLensColumn.Date, + ...(breakdownFieldName + ? { splitAccessor: DatasetQualityLensColumn.Breakdown } + : { + yConfig: [ + { + forAccessor: DatasetQualityLensColumn.FailedDocs, + color, + }, + ], + }), + }, + ], + legend: { + isVisible: true, + position: 'right', + legendSize: 'large', + shouldTruncate: true, + }, + preferredSeriesType: 'line', + tickLabelsVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + valueLabels: 'hide', + yLeftExtent: { + mode: 'custom', + lowerBound: 0, + upperBound: undefined, + }, + }, + }, + } as TypedLensByValueInput['attributes']; +} + +function getAdHocDataViewState(id: string, dataStream: string, title: string) { + return { + internalReferences: [ + { + id, + name: 'indexpattern-datasource-current-indexpattern', + type: 'index-pattern', + }, + { + id, + name: 'indexpattern-datasource-layer-layer1', + type: 'index-pattern', + }, + ], + adHocDataViews: { + [id]: { + id, + title: dataStream, + timeFieldName: '@timestamp', + sourceFilters: [], + fieldFormats: {}, + runtimeFieldMap: {}, + fieldAttrs: {}, + allowNoIndex: false, + name: title, + }, + }, + }; +} + +function getChartColumns(breakdownField?: string): Record { + return { + [DatasetQualityLensColumn.Date]: { + dataType: 'date', + isBucketed: true, + label: '@timestamp', + operationType: 'date_histogram', + params: { + interval: 'auto', + }, + scale: 'interval', + sourceField: '@timestamp', + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.CountFailed]: { + label: '', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + filter: { + query: '', + language: 'kuery', + }, + params: { + emptyAsNull: false, + }, + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.CountTotal]: { + label: '', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + params: { + emptyAsNull: false, + }, + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.Math]: { + label: '', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'divide', + args: ['count_ignored', 'count_total'], + location: { + min: 0, + max: 34, + }, + text: "count(kql='') / count()", + }, + }, + references: ['count_ignored', 'count_total'], + customLabel: true, + } as GenericIndexPatternColumn, + [DatasetQualityLensColumn.FailedDocs]: { + label: flyoutFailedDocsPercentageText, + customLabel: true, + operationType: 'formula', + dataType: 'number', + references: [DatasetQualityLensColumn.Math], + isBucketed: false, + params: { + formula: "count(kql='') / count()", + format: { + id: 'percent', + params: { + decimals: 3, + }, + }, + isFormulaBroken: false, + }, + } as GenericIndexPatternColumn, + ...(breakdownField + ? { + [DatasetQualityLensColumn.Breakdown]: { + dataType: 'number', + isBucketed: true, + label: getFlyoutDegradedDocsTopNText(MAX_BREAKDOWN_SERIES, breakdownField), + operationType: 'terms', + scale: 'ordinal', + sourceField: breakdownField, + params: { + size: MAX_BREAKDOWN_SERIES, + orderBy: { + type: 'alphabetical', + fallback: true, + }, + orderDirection: 'desc', + otherBucket: true, + missingBucket: true, + parentFormat: { + id: 'terms', + }, + }, + } as GenericIndexPatternColumn, + } + : {}), + }; +} + +const getFlyoutDegradedDocsTopNText = (count: number, fieldName: string) => + i18n.translate('xpack.datasetQuality.details.degradedDocsTopNValues', { + defaultMessage: 'Top {count} values of {fieldName}', + values: { count, fieldName }, + description: + 'Tooltip label for the top N values of a field in the degraded documents trend chart.', + }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 897efb821ff64..13344ceb91767 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -6,10 +6,12 @@ */ import React from 'react'; -import { EuiFlexGroup } from '@elastic/eui'; +import { EuiCode, EuiFlexGroup } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { Panel, PanelIndicator } from './panel'; import { overviewPanelDatasetQualityIndicatorDegradedDocs, + overviewPanelDatasetQualityIndicatorFailedDocs, overviewPanelDocumentsIndicatorSize, overviewPanelDocumentsIndicatorTotalCount, overviewPanelResourcesIndicatorServices, @@ -21,6 +23,27 @@ import { import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel'; import { DatasetQualityIndicator } from '../../../quality_indicator'; +const degradedDocsTooltip = ( + + _ignored + + ), + }} + /> +); + +const failedDocsColumnTooltip = ( + +); + // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function Summary() { @@ -32,6 +55,7 @@ export default function Summary() { totalServicesCount, totalHostsCount, totalDegradedDocsCount, + totalFailedDocsCount, quality, } = useOverviewSummaryPanel(); return ( @@ -75,6 +99,13 @@ export default function Summary() { label={overviewPanelDatasetQualityIndicatorDegradedDocs} value={totalDegradedDocsCount} isLoading={isSummaryPanelLoading} + tooltip={degradedDocsTooltip} + /> + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx index e03e0957b5a52..f11bb14be29fd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/panel.tsx @@ -6,7 +6,15 @@ */ import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSkeletonTitle, EuiText } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiPanel, + EuiSkeletonTitle, + EuiText, + EuiToolTip, +} from '@elastic/eui'; import { css } from '@emotion/react'; import { euiThemeVars } from '@kbn/ui-theme'; import { PrivilegesWarningIconWrapper } from '../../../common'; @@ -71,11 +79,13 @@ export function Panel({ export function PanelIndicator({ label, value, + tooltip, isLoading, userHasPrivilege = true, }: { label: string; value: string | number; + tooltip?: React.ReactElement; isLoading: boolean; userHasPrivilege?: boolean; }) { @@ -85,9 +95,23 @@ export function PanelIndicator({ ) : ( <> - - {label} - + {tooltip ? ( + + + {`${label} `} + + + + ) : ( + + {label} + + )} { + const { euiTheme } = useEuiTheme(); + const { + services: { lens }, + } = useKibanaContextForPlugin(); + const { + service, + dataStream, + datasetDetails, + timeRange, + breakdownField, + integrationDetails, + isBreakdownFieldAsserted, + } = useDatasetQualityDetailsState(); + + const { + trackDatasetDetailsBreakdownFieldChanged, + trackDetailsNavigated, + navigationTargets, + navigationSources, + } = useDatasetDetailsTelemetry(); + + const [isChartLoading, setIsChartLoading] = useState(undefined); + const [attributes, setAttributes] = useState | undefined>( + undefined + ); + + const { dataView } = useCreateDataView({ + indexPatternString: getDataViewIndexPattern(dataStream), + }); + + const breakdownDataViewField = useMemo( + () => getDataViewField(dataView, breakdownField), + [breakdownField, dataView] + ); + + const handleChartLoading = (isLoading: boolean) => { + setIsChartLoading(isLoading); + }; + + const handleBreakdownFieldChange = useCallback( + (field: DataViewField | undefined) => { + service.send({ + type: 'BREAKDOWN_FIELD_CHANGE', + breakdownField: field?.name, + }); + }, + [service] + ); + + useEffect(() => { + if (isBreakdownFieldAsserted) trackDatasetDetailsBreakdownFieldChanged(); + }, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]); + + useEffect(() => { + const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW; + const datasetTitle = + integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; + + const lensAttributes = getLensAttributes({ + color: euiTheme.colors.danger, + dataStream: dataStreamName, + datasetTitle, + breakdownFieldName: breakdownDataViewField?.name, + }); + setAttributes(lensAttributes); + }, [ + breakdownDataViewField?.name, + euiTheme.colors.danger, + setAttributes, + dataStream, + integrationDetails?.integration?.datasets, + datasetDetails.name, + ]); + + const openInLensCallback = useCallback(() => { + if (attributes) { + trackDetailsNavigated(navigationTargets.Lens, navigationSources.Chart); + lens.navigateToPrefilledEditor({ + id: '', + timeRange, + attributes, + }); + } + }, [ + attributes, + lens, + navigationSources.Chart, + navigationTargets.Lens, + timeRange, + trackDetailsNavigated, + ]); + + const getOpenInLensAction = useMemo(() => { + return { + id: ACTION_OPEN_IN_LENS, + type: 'link', + order: 17, + getDisplayName(): string { + return openInLensText; + }, + getIconType(): string { + return 'visArea'; + }, + async isCompatible(): Promise { + return true; + }, + async execute(): Promise { + return openInLensCallback(); + }, + }; + }, [openInLensCallback]); + + const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({ + query: { language: 'kuery', query: '_ignored:*' }, + navigationSource: navigationSources.Chart, + }); + + const redirectLinkProps = useRedirectLink({ + dataStreamStat: datasetDetails, + query: { language: 'kuery', query: '_ignored:*' }, + timeRangeConfig: timeRange, + breakdownField: breakdownDataViewField?.name, + sendTelemetry, + }); + + const getOpenInLogsExplorerAction = useMemo(() => { + return { + id: ACTION_EXPLORE_IN_LOGS_EXPLORER, + type: 'link', + getDisplayName(): string { + return redirectLinkProps?.isLogsExplorerAvailable + ? exploreDataInLogsExplorerText + : exploreDataInDiscoverText; + }, + getHref: async () => { + return redirectLinkProps.linkProps.href; + }, + getIconType(): string | undefined { + return 'visTable'; + }, + async isCompatible(): Promise { + return true; + }, + async execute(): Promise { + return redirectLinkProps.navigate(); + }, + order: 18, + }; + }, [redirectLinkProps]); + + const extraActions: Action[] = [getOpenInLensAction, getOpenInLogsExplorerAction]; + + const breakdown = useMemo(() => { + return { + dataViewField: breakdownDataViewField, + fieldSupportsBreakdown: breakdownDataViewField + ? fieldSupportsBreakdown(breakdownDataViewField) + : true, + onChange: handleBreakdownFieldChange, + }; + }, [breakdownDataViewField, handleBreakdownFieldChange]); + + return { + attributes, + dataView, + breakdown, + extraActions, + isChartLoading, + onChartLoading: handleChartLoading, + setAttributes, + setIsChartLoading, + }; +}; + +function getDataViewIndexPattern(dataStream: string | undefined) { + return dataStream ?? DEFAULT_LOGS_DATA_VIEW; +} + +function getDataViewField(dataView: DataView | undefined, fieldName: string | undefined) { + return fieldName && dataView + ? dataView.fields.find((field) => field.name === fieldName) + : undefined; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts index 43cf6923075ee..98e7aba30d493 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts @@ -55,12 +55,19 @@ export const useOverviewSummaryPanel = () => { NUMBER_FORMAT ); + const totalFailedDocsCount = formatNumber(dataStreamDetails?.failedDocsCount ?? 0, NUMBER_FORMAT); + const degradedPercentage = - Number(totalDocsCount) > 0 - ? (Number(totalDegradedDocsCount) / Number(totalDocsCount)) * 100 + (dataStreamDetails.docsCount ?? 0) > 0 + ? ((dataStreamDetails?.degradedDocsCount ?? 0) / dataStreamDetails.docsCount!) * 100 + : 0; + + const failedPercentage = + (dataStreamDetails.docsCount ?? 0) > 0 + ? ((dataStreamDetails?.failedDocsCount ?? 0) / dataStreamDetails.docsCount!) * 100 : 0; - const quality = mapPercentageToQuality(degradedPercentage); + const quality = mapPercentageToQuality([degradedPercentage, failedPercentage]); return { totalDocsCount, @@ -70,6 +77,7 @@ export const useOverviewSummaryPanel = () => { totalHostsCount, isSummaryPanelLoading, totalDegradedDocsCount, + totalFailedDocsCount, quality, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 288eff11b92a8..c9212909fcc8c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -21,6 +21,7 @@ import { createDatasetQualityESClient } from '../../../utils'; import { dataStreamService, datasetQualityPrivileges } from '../../../services'; import { getDataStreams } from '../get_data_streams'; import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats'; +import { getFailedDocsPaginated } from '../get_failed_docs'; export async function getDataStreamSettings({ esClient, @@ -92,6 +93,14 @@ export async function getDataStreamDetails({ end ); + const failedDocs = await getFailedDocsPaginated({ + esClient: esClientAsCurrentUser, + types: [], + datasetQuery: dataStream, + start, + end, + }); + const avgDocSizeInBytes = hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0 ? isServerless @@ -103,6 +112,7 @@ export async function getDataStreamDetails({ return { ...dataStreamSummaryStats, + failedDocsCount: failedDocs[0]?.count, sizeBytes, lastActivity: esDataStream?.lastActivity, userPrivileges: { diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts new file mode 100644 index 0000000000000..170da8def9439 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/failed_docs.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { APIClientRequestParamsOf } from '@kbn/dataset-quality-plugin/common/rest'; +import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { merge } from 'lodash'; +import rison from '@kbn/rison'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('synthtrace'); + const from = '2024-09-20T11:00:00.000Z'; + const to = '2024-09-20T11:01:00.000Z'; + const dataStreamType = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${dataStreamType}-${dataset}-${namespace}`; + const syntheticsDataStreamName = `${dataStreamType}-${syntheticsDataset}-${namespace}`; + + const endpoint = 'GET /internal/dataset_quality/data_streams/failed_docs'; + type ApiParams = APIClientRequestParamsOf['params']['query']; + type DataStreamType = ApiParams['types'][0]; + + const processors = [ + { + script: { + tag: 'normalize log level', + lang: 'painless', + source: ` + String level = ctx['log.level']; + if ('0'.equals(level)) { + ctx['log.level'] = 'info'; + } else if ('1'.equals(level)) { + ctx['log.level'] = 'debug'; + } else if ('2'.equals(level)) { + ctx['log.level'] = 'warning'; + } else if ('3'.equals(level)) { + ctx['log.level'] = 'error'; + } else { + throw new Exception("Not a valid log level"); + } + `, + }, + }, + ]; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { types = rison.encodeArray(['logs']), start, end }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: Omit & { types?: DataStreamType[] }; + }) { + return roleScopedSupertestWithCookieCredentials + .get(`/internal/dataset_quality/data_streams/failed_docs`) + .query({ + types, + start, + end, + }); + } + + describe('DataStream failed docs', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + let synthtraceLogsEsClient: LogsSynthtraceEsClient; + + before(async () => { + synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient(); + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + + await synthtraceLogsEsClient.createCustomPipeline(processors); + await synthtraceLogsEsClient.updateIndexTemplate('logs', (template) => { + const next = { + name: 'logs', + data_stream: { + failure_store: true, + }, + }; + + return merge({}, template, next); + }); + await synthtraceLogsEsClient.index([ + timerange(from, to) + .interval('1m') + .rate(1) + .generator((timestamp) => [ + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .logLevel('0') + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .logLevel('5') + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + ]), + ]); + }); + + after(async () => { + await synthtraceLogsEsClient.clean(); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('returns number of failed documents per DataStream', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: from, + end: to, + }, + }); + + expect(resp.body.failedDocs.length).to.be(1); + expect(resp.body.totalDocs[0].dataset).to.be(syntheticsDataStreamName); + expect(resp.body.totalDocs[0].count).to.be(1); + }); + + it('returns empty when all documents are outside timeRange', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + start: '2024-09-21T11:00:00.000Z', + end: '2024-09-21T11:01:00.000Z', + }, + }); + + expect(resp.body.failedDocs.length).to.be(0); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0481e882aee6e..5f9ce787a44dd 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -19,5 +19,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./degraded_fields')); loadTestFile(require.resolve('./data_stream_details')); loadTestFile(require.resolve('./degraded_field_values')); + loadTestFile(require.resolve('./failed_docs')); }); }