diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/common/insufficient_privileges.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/common/insufficient_privileges.tsx index de9c99a7eda9..596a220a5b13 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/common/insufficient_privileges.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/common/insufficient_privileges.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback } from 'react'; +import React from 'react'; import useToggle from 'react-use/lib/useToggle'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -54,7 +54,7 @@ export const PrivilegesWarningIconWrapper = ({ }) => { const [isPopoverOpen, togglePopover] = useToggle(false); - const handleButtonClick = useCallback(() => togglePopover(true), [togglePopover]); + const handleButtonClick = togglePopover; if (hasPrivileges) { return <>{children}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_activity.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_activity.tsx index 0596e7b05c74..02cd22481bef 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_activity.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_activity.tsx @@ -16,8 +16,7 @@ import { import { DataPlaceholder } from './data_placeholder'; export function DatasetsActivity() { - const { datasetsActivity, isDatasetsActivityLoading, isUserAuthorizedForDataset } = - useSummaryPanelContext(); + const { datasetsActivity, isDatasetsActivityLoading } = useSummaryPanelContext(); const text = `${datasetsActivity.active} ${tableSummaryOfText} ${datasetsActivity.total}`; return ( @@ -26,7 +25,7 @@ export function DatasetsActivity() { tooltip={summaryPanelDatasetsActivityTooltipText} value={text} isLoading={isDatasetsActivityLoading} - isUserAuthorizedForDataset={isUserAuthorizedForDataset} + isUserAuthorizedForDataset={true} /> ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 98c535bd0921..da6bbf1628d1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -300,46 +300,37 @@ export const getDatasetQualityTableColumns = ({ ), width: '140px', }, - ...(canUserMonitorDataset && canUserMonitorAnyDataStream - ? [ - { - name: ( - - {lastActivityColumnName} - - ), - field: 'lastActivity', - render: (timestamp: number, { userPrivileges, title }: DataStreamStat) => ( - - - {!isActiveDataset(timestamp) ? ( - - {inactiveDatasetActivityColumnDescription} - - - - - ) : ( - fieldFormats - .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) - .convert(timestamp) - )} - - - ), - width: '300px', - sortable: true, - }, - ] - : []), + { + name: ( + + {lastActivityColumnName} + + ), + field: 'lastActivity', + render: (timestamp: number) => ( + + {!isActiveDataset(timestamp) ? ( + + {inactiveDatasetActivityColumnDescription} + + + + + ) : ( + fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE]) + .convert(timestamp) + )} + + ), + width: '300px', + sortable: true, + }, { name: actionsColumnName, render: (dataStreamStat: DataStreamStat) => ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index e2067dedd26d..4aeeb1087ba8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -19,7 +19,7 @@ import { _IGNORED } from '../../../../common/es_fields'; import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_types'; import { createDatasetQualityESClient } from '../../../utils'; import { dataStreamService, datasetQualityPrivileges } from '../../../services'; -import { getDataStreamsStats } from '../get_data_streams_stats'; +import { getDataStreams } from '../get_data_streams'; export async function getDataStreamSettings({ esClient, @@ -49,13 +49,13 @@ export async function getDataStreamDetails({ dataStream, start, end, - sizeStatsAvailable = true, + isServerless, }: { esClient: ElasticsearchClient; dataStream: string; start: number; end: number; - sizeStatsAvailable?: boolean; // Only Needed to determine whether `_stats` endpoint is available https://github.com/elastic/kibana/issues/178954 + isServerless: boolean; }): Promise { throwIfInvalidDataStreamParams(dataStream); @@ -63,14 +63,13 @@ export async function getDataStreamDetails({ await datasetQualityPrivileges.getHasIndexPrivileges(esClient, [dataStream], ['monitor']) )[dataStream]; - const lastActivity = hasAccessToDataStream + const esDataStream = hasAccessToDataStream ? ( - await getDataStreamsStats({ + await getDataStreams({ esClient, - dataStreams: [dataStream], - sizeStatsAvailable, + datasetQuery: dataStream, }) - ).items[0]?.lastActivity + ).dataStreams[0] : undefined; try { @@ -82,17 +81,17 @@ export async function getDataStreamDetails({ ); const whenSizeStatsNotAvailable = NaN; // This will indicate size cannot be calculated - const avgDocSizeInBytes = sizeStatsAvailable - ? hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0 - ? await getAvgDocSizeInBytes(esClient, dataStream) - : 0 - : whenSizeStatsNotAvailable; + const avgDocSizeInBytes = isServerless + ? whenSizeStatsNotAvailable + : hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0 + ? await getAvgDocSizeInBytes(esClient, dataStream) + : 0; const sizeBytes = Math.ceil(avgDocSizeInBytes * dataStreamSummaryStats.docsCount); return { ...dataStreamSummaryStats, sizeBytes, - lastActivity, + lastActivity: esDataStream?.lastActivity, userPrivileges: { canMonitor: hasAccessToDataStream, }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts index 907c0c0fdc05..4f1c4ea84570 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts @@ -40,8 +40,22 @@ describe('getDataStreams', () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); const result = await getDataStreams({ esClient: esClientMock, - types: ['logs'], - datasetQuery: 'nginx-*', + types: ['logs', 'metrics'], + uncategorisedOnly: true, + }); + expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith( + expect.anything(), + 'logs-*-*,metrics-*-*' + ); + + expect(result.datasetUserPrivileges.canMonitor).toBe(true); + }); + + it('Passes datasetQuery parameter to the DataStreamService', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + const result = await getDataStreams({ + esClient: esClientMock, + datasetQuery: 'logs-nginx-*', uncategorisedOnly: true, }); expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith( @@ -58,20 +72,18 @@ describe('getDataStreams', () => { const results = await getDataStreams({ esClient: esClientMock, types: ['logs'], - datasetQuery: 'nginx-*', uncategorisedOnly: true, }); - expect(results.items.length).toBe(1); + expect(results.dataStreams.length).toBe(1); }); it('Returns the correct number of results when false', async () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); const results = await getDataStreams({ esClient: esClientMock, types: ['logs'], - datasetQuery: 'nginx-*', uncategorisedOnly: false, }); - expect(results.items.length).toBe(5); + expect(results.dataStreams.length).toBe(5); }); }); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts index 853a0cf208b4..f2a4f458560e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts @@ -12,18 +12,20 @@ import { dataStreamService, datasetQualityPrivileges } from '../../../services'; export async function getDataStreams(options: { esClient: ElasticsearchClient; - types: DataStreamType[]; + types?: DataStreamType[]; datasetQuery?: string; - uncategorisedOnly: boolean; + uncategorisedOnly?: boolean; }) { - const { esClient, types, datasetQuery, uncategorisedOnly } = options; + const { esClient, types = [], datasetQuery, uncategorisedOnly } = options; - const datasetNames = types.map((type) => - streamPartsToIndexPattern({ - typePattern: type, - datasetPattern: datasetQuery ? `${datasetQuery}` : '*-*', - }) - ); + const datasetNames = datasetQuery + ? [datasetQuery] + : types.map((type) => + streamPartsToIndexPattern({ + typePattern: type, + datasetPattern: '*-*', + }) + ); const datasetUserPrivileges = await datasetQualityPrivileges.getDatasetPrivileges( esClient, @@ -32,7 +34,7 @@ export async function getDataStreams(options: { if (!datasetUserPrivileges.canMonitor) { return { - items: [], + dataStreams: [], datasetUserPrivileges, }; } @@ -59,13 +61,15 @@ export async function getDataStreams(options: { const mappedDataStreams = filteredDataStreams.map((dataStream) => ({ name: dataStream.name, integration: dataStream._meta?.package?.name, + // @ts-expect-error + lastActivity: dataStream.maximum_timestamp, userPrivileges: { canMonitor: dataStreamsPrivileges[dataStream.name], }, })); return { - items: mappedDataStreams, + dataStreams: mappedDataStreams, datasetUserPrivileges, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts index 8c7878f24486..e99c5866c2f9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts @@ -12,42 +12,31 @@ import { indexStatsService } from '../../../services'; export async function getDataStreamsStats({ esClient, dataStreams, - sizeStatsAvailable = true, }: { esClient: ElasticsearchClient; dataStreams: string[]; - sizeStatsAvailable?: boolean; // Only Needed to determine whether `_stats` endpoint is available https://github.com/elastic/kibana/issues/178954 -}) { +}): Promise> { if (!dataStreams.length) { - return { - items: [], - }; + return {}; } const matchingDataStreamsStats = dataStreamService.getStreamsStats(esClient, dataStreams); - - const indicesDocsCount = sizeStatsAvailable - ? indexStatsService.getIndicesDocCounts(esClient, dataStreams) - : Promise.resolve(null); + const indicesDocsCount = indexStatsService.getIndicesDocCounts(esClient, dataStreams); const [indicesDocsCountStats, dataStreamsStats] = await Promise.all([ indicesDocsCount, matchingDataStreamsStats, ]); - const mappedDataStreams = dataStreamsStats.map((dataStream) => { - return { - name: dataStream.data_stream, - size: dataStream.store_size?.toString(), - sizeBytes: dataStream.store_size_bytes, - lastActivity: dataStream.maximum_timestamp, - totalDocs: sizeStatsAvailable - ? indicesDocsCountStats!.docsCountPerDataStream[dataStream.data_stream] || 0 - : null, - }; - }); - - return { - items: mappedDataStreams, - }; + return dataStreamsStats.reduce( + (acc, dataStream) => ({ + ...acc, + [dataStream.data_stream]: { + size: dataStream.store_size!.toString(), + sizeBytes: dataStream.store_size_bytes, + totalDocs: indicesDocsCountStats!.docsCountPerDataStream[dataStream.data_stream], + }, + }), + {} + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index dd0e278acb94..c23b917a18e8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -6,7 +6,6 @@ */ import * as t from 'io-ts'; -import { keyBy, merge, values } from 'lodash'; import { DataStreamDetails, DataStreamSettings, @@ -47,30 +46,37 @@ const statsRoute = createDatasetQualityServerRoute({ }> { const { context, params, getEsCapabilities } = resources; const coreContext = await context.core; - const sizeStatsAvailable = !(await getEsCapabilities()).serverless; + const isServerless = (await getEsCapabilities()).serverless; // Query datastreams as the current user as the Kibana internal user may not have all the required permissions const esClient = coreContext.elasticsearch.client.asCurrentUser; - const { items, datasetUserPrivileges } = await getDataStreams({ + const { dataStreams, datasetUserPrivileges } = await getDataStreams({ esClient, ...params.query, uncategorisedOnly: false, }); - const privilegedDataStreams = items.filter((stream) => { - return stream.userPrivileges.canMonitor; + const privilegedDataStreams = dataStreams.filter((dataStream) => { + return dataStream.userPrivileges.canMonitor; }); - const dataStreamsStats = await getDataStreamsStats({ - esClient, - dataStreams: privilegedDataStreams.map((stream) => stream.name), - sizeStatsAvailable, - }); + const dataStreamsStats = isServerless + ? {} + : await getDataStreamsStats({ + esClient, + dataStreams: privilegedDataStreams.map((stream) => stream.name), + }); return { datasetUserPrivileges, - dataStreamsStats: values(merge(keyBy(items, 'name'), keyBy(dataStreamsStats.items, 'name'))), + dataStreamsStats: dataStreams.map((dataStream: DataStreamStat) => { + dataStream.size = dataStreamsStats[dataStream.name]?.size; + dataStream.sizeBytes = dataStreamsStats[dataStream.name]?.sizeBytes; + dataStream.totalDocs = dataStreamsStats[dataStream.name]?.totalDocs; + + return dataStream; + }), }; }, }); @@ -268,13 +274,13 @@ const dataStreamDetailsRoute = createDatasetQualityServerRoute({ // Query datastreams as the current user as the Kibana internal user may not have all the required permissions const esClient = coreContext.elasticsearch.client.asCurrentUser; - const sizeStatsAvailable = !(await getEsCapabilities()).serverless; + const isServerless = (await getEsCapabilities()).serverless; const dataStreamDetails = await getDataStreamDetails({ esClient, dataStream, start, end, - sizeStatsAvailable, + isServerless, }); return dataStreamDetails; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/services/data_stream.ts b/x-pack/plugins/observability_solution/dataset_quality/server/services/data_stream.ts index 0446e27953af..16b283d583fd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/services/data_stream.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/services/data_stream.ts @@ -19,6 +19,8 @@ class DataStreamService { try { const { data_streams: dataStreamsInfo } = await esClient.indices.getDataStream({ name: datasetName, + // @ts-expect-error + verbose: true, }); return dataStreamsInfo; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_privileges.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_privileges.ts index c94a08394606..949d42dbd31c 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_privileges.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_privileges.ts @@ -20,7 +20,6 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const security = getService('security'); const synthtrace = getService('logSynthtraceEsClient'); const testSubjects = getService('testSubjects'); - const find = getService('find'); const to = new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(); const apacheAccessDatasetName = 'apache.access'; @@ -111,26 +110,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await synthtrace.clean(); }); - it('Active and Estimated data are not available due to underprivileged user', async () => { - await testSubjects.existOrFail( - `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-${PageObjects.datasetQuality.texts.activeDatasets}` - ); + it('Estimated data are not available due to underprivileged user', async () => { await testSubjects.existOrFail( `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-${PageObjects.datasetQuality.texts.estimatedData}` ); }); - it('"Show inactive datasets" is hidden when lastActivity is not available', async () => { - await find.waitForDeletedByCssSelector( - PageObjects.datasetQuality.selectors.showInactiveDatasetsNamesSwitch - ); - }); - - it('does not show size and last activity columns for underprivileged data stream', async () => { + it('does not show size column for underprivileged data stream', async () => { const cols = await PageObjects.datasetQuality.getDatasetTableHeaderTexts(); expect(cols).to.not.contain('Size'); - expect(cols).to.not.contain('Last Activity'); }); }); @@ -149,27 +138,21 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await synthtrace.clean(); }); - it('shows underprivileged warning when size and last activity cannot be accessed for some data streams', async () => { + it('shows underprivileged warning when size cannot be accessed for some data streams', async () => { await PageObjects.datasetQuality.refreshTable(); const datasetWithMonitorPrivilege = apacheAccessDatasetHumanName; const datasetWithoutMonitorPrivilege = 'synth.1'; - // "Size" and "Last Activity" should be available for `apacheAccessDatasetName` + // "Size" should be available for `apacheAccessDatasetName` await testSubjects.missingOrFail( `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-sizeBytes-${datasetWithMonitorPrivilege}` ); - await testSubjects.missingOrFail( - `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-lastActivity-${datasetWithMonitorPrivilege}` - ); - // "Size" and "Last Activity" should not be available for `datasetWithoutMonitorPrivilege` + // "Size" should not be available for `datasetWithoutMonitorPrivilege` await testSubjects.existOrFail( `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-sizeBytes-${datasetWithoutMonitorPrivilege}` ); - await testSubjects.existOrFail( - `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-lastActivity-${datasetWithoutMonitorPrivilege}` - ); }); it('Details page shows insufficient privileges warning for underprivileged data stream', async () => { diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_privileges.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_privileges.ts index 907e24ff8fa0..37115f558f65 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_privileges.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_privileges.ts @@ -18,9 +18,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'svlCommonNavigation', 'svlCommonPage', ]); - const testSubjects = getService('testSubjects'); const synthtrace = getService('svlLogsSynthtraceClient'); - const find = getService('find'); const to = '2024-01-01T12:00:00.000Z'; describe('Dataset quality user privileges', function () { @@ -37,29 +35,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await synthtrace.clean(); }); - it('Active Datasets stat is not available due to underprivileged user', async () => { - await testSubjects.existOrFail( - `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityInsufficientPrivileges}-${PageObjects.datasetQuality.texts.activeDatasets}` - ); - }); - - it('"Show inactive datasets" is hidden when lastActivity is not available', async () => { - await find.waitForDeletedByCssSelector( - PageObjects.datasetQuality.selectors.showInactiveDatasetsNamesSwitch - ); - }); - - it('does not show last activity column for underprivileged data stream', async () => { - const cols = await PageObjects.datasetQuality.getDatasetTableHeaderTexts(); - - expect(cols).to.not.contain('Last Activity'); - }); - it('does not show size and last activity columns for underprivileged data stream', async () => { const cols = await PageObjects.datasetQuality.getDatasetTableHeaderTexts(); expect(cols).to.not.contain('Size'); - expect(cols).to.not.contain('Last Activity'); }); }); }