diff --git a/x-pack/plugins/dataset_quality/common/api_types.ts b/x-pack/plugins/dataset_quality/common/api_types.ts index 4f9936d65e014..70a5c3e597148 100644 --- a/x-pack/plugins/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/dataset_quality/common/api_types.ts @@ -7,7 +7,7 @@ import * as rt from 'io-ts'; -export const datasetStatRt = rt.intersection([ +export const dataStreamStatRt = rt.intersection([ rt.type({ name: rt.string, }), @@ -19,6 +19,8 @@ export const datasetStatRt = rt.intersection([ }), ]); +export type DataStreamStat = rt.TypeOf; + export const integrationIconRt = rt.intersection([ rt.type({ path: rt.string, @@ -39,9 +41,12 @@ export const integrationRt = rt.intersection([ title: rt.string, version: rt.string, icons: rt.array(integrationIconRt), + datasets: rt.record(rt.string, rt.string), }), ]); +export type Integration = rt.TypeOf; + export const degradedDocsRt = rt.type({ dataset: rt.string, percentage: rt.number, @@ -52,7 +57,7 @@ export type DegradedDocs = rt.TypeOf; export const getDataStreamsStatsResponseRt = rt.exact( rt.intersection([ rt.type({ - dataStreamsStats: rt.array(datasetStatRt), + dataStreamsStats: rt.array(dataStreamStatRt), }), rt.type({ integrations: rt.array(integrationRt), diff --git a/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 2456284eb3cdc..88b6ed8d5c68b 100644 --- a/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -10,16 +10,18 @@ import { DataStreamStatType, IntegrationType } from './types'; export class DataStreamStat { name: DataStreamStatType['name']; + namespace: string; title: string; size?: DataStreamStatType['size']; - sizeBytes?: DataStreamStatType['size_bytes']; - lastActivity?: DataStreamStatType['last_activity']; + sizeBytes?: DataStreamStatType['sizeBytes']; + lastActivity?: DataStreamStatType['lastActivity']; integration?: IntegrationType; degradedDocs?: number; private constructor(dataStreamStat: DataStreamStat) { this.name = dataStreamStat.name; this.title = dataStreamStat.title ?? dataStreamStat.name; + this.namespace = dataStreamStat.namespace; this.size = dataStreamStat.size; this.sizeBytes = dataStreamStat.sizeBytes; this.lastActivity = dataStreamStat.lastActivity; @@ -32,10 +34,11 @@ export class DataStreamStat { const dataStreamStatProps = { name: dataStreamStat.name, - title: `${dataset}-${namespace}`, + title: dataStreamStat.integration?.datasets?.[dataset] ?? dataset, + namespace, size: dataStreamStat.size, - sizeBytes: dataStreamStat.size_bytes, - lastActivity: dataStreamStat.last_activity, + sizeBytes: dataStreamStat.sizeBytes, + lastActivity: dataStreamStat.lastActivity, integration: dataStreamStat.integration ? Integration.create(dataStreamStat.integration) : undefined, diff --git a/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx b/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx index 94223e10f316f..0bdfcf0d28770 100644 --- a/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx +++ b/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx @@ -7,6 +7,7 @@ import React from 'react'; import { + EuiBadge, EuiBasicTableColumn, EuiCode, EuiFlexGroup, @@ -33,6 +34,10 @@ const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', { defaultMessage: 'Dataset Name', }); +const namespaceColumnName = i18n.translate('xpack.datasetQuality.namespaceColumnName', { + defaultMessage: 'Namespace', +}); + const sizeColumnName = i18n.translate('xpack.datasetQuality.sizeColumnName', { defaultMessage: 'Size', }); @@ -122,6 +127,14 @@ export const getDatasetQualitTableColumns = ({ ); }, }, + { + name: namespaceColumnName, + field: 'namespace', + sortable: true, + render: (_, dataStreamStat: DataStreamStat) => ( + {dataStreamStat.namespace} + ), + }, { name: sizeColumnName, field: 'size', diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts index 6154e3bc11a24..b79b4eeec0116 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts @@ -6,8 +6,8 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; +import { DataStreamTypes } from '../../../types/default_api_types'; import { dataStreamService } from '../../../services'; -import { DataStreamTypes } from '../../../types/data_stream'; export async function getDataStreams(options: { esClient: ElasticsearchClient; diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts index 830c3b162573f..a0104afc1d5fe 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts @@ -79,32 +79,32 @@ describe('getDataStreams', () => { { name: 'logs-elastic_agent-default', size: '1gb', - size_bytes: 1170805528, - last_activity: 1698916071000, + sizeBytes: 1170805528, + lastActivity: 1698916071000, }, { name: 'logs-elastic_agent.filebeat-default', size: '1.3mb', - size_bytes: 1459100, - last_activity: 1698902209996, + sizeBytes: 1459100, + lastActivity: 1698902209996, }, { name: 'logs-elastic_agent.fleet_server-default', size: '2.9mb', - size_bytes: 3052148, - last_activity: 1698914110010, + sizeBytes: 3052148, + lastActivity: 1698914110010, }, { name: 'logs-elastic_agent.metricbeat-default', size: '1.6mb', - size_bytes: 1704807, - last_activity: 1698672046707, + sizeBytes: 1704807, + lastActivity: 1698672046707, }, { name: 'logs-test.test-default', size: '6.2mb', - size_bytes: 6570447, - last_activity: 1698913802000, + sizeBytes: 6570447, + lastActivity: 1698913802000, }, ]); }); diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts index 9ec252d096357..a78f2fec53e29 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts @@ -6,8 +6,8 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; +import { DataStreamTypes } from '../../../types/default_api_types'; import { dataStreamService } from '../../../services'; -import { DataStreamTypes } from '../../../types/data_stream'; export async function getDataStreamsStats(options: { esClient: ElasticsearchClient; @@ -24,9 +24,9 @@ export async function getDataStreamsStats(options: { const mappedDataStreams = matchingDataStreamsStats.map((dataStream) => { return { name: dataStream.data_stream, - size: dataStream.store_size, - size_bytes: dataStream.store_size_bytes, - last_activity: dataStream.maximum_timestamp, + size: dataStream.store_size?.toString(), + sizeBytes: dataStream.store_size_bytes, + lastActivity: dataStream.maximum_timestamp, }; }); diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts index 4bbe01d219322..1af5a603f3638 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts @@ -14,7 +14,7 @@ import { DATA_STREAM_TYPE, _IGNORED, } from '../../../common/es_fields'; -import { DataStreamTypes } from '../../types/data_stream'; +import { DataStreamTypes } from '../../types/default_api_types'; import { createDatasetQualityESClient, wildcardQuery } from '../../utils'; export async function getDegradedDocsPaginated(options: { diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_integrations.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_integrations.ts new file mode 100644 index 0000000000000..7871571b607b6 --- /dev/null +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_integrations.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PackageClient } from '@kbn/fleet-plugin/server'; +import { DataStreamStat, Integration } from '../../../common/api_types'; + +export async function getIntegrations(options: { + packageClient: PackageClient; + dataStreams: DataStreamStat[]; +}): Promise { + const { packageClient, dataStreams } = options; + + const packages = await packageClient.getPackages(); + const installedPackages = dataStreams.map((item) => item.integration); + + return Promise.all( + packages + .filter((pkg) => installedPackages.includes(pkg.name)) + .map(async (p) => ({ + name: p.name, + title: p.title, + version: p.version, + icons: p.icons, + datasets: await getDatasets({ + packageClient, + name: p.name, + version: p.version, + }), + })) + ); +} + +const getDatasets = async (options: { + packageClient: PackageClient; + name: string; + version: string; +}) => { + const { packageClient, name, version } = options; + + const pkg = await packageClient.getPackage(name, version); + + return pkg.packageInfo.data_streams?.reduce( + (acc, curr) => ({ + ...acc, + [curr.dataset]: curr.title, + }), + {} + ); +}; diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts index 7be8c102060d6..4f95331d97651 100644 --- a/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts @@ -7,20 +7,19 @@ import * as t from 'io-ts'; import { keyBy, merge, values } from 'lodash'; -import { DataStreamStat } from '../../types/data_stream'; -import { dataStreamTypesRt, rangeRt } from '../../types/default_api_types'; -import { Integration } from '../../types/integration'; +import { typeRt, rangeRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; -import { DegradedDocs } from '../../../common/api_types'; +import { DegradedDocs, DataStreamStat, Integration } from '../../../common/api_types'; +import { getIntegrations } from './get_integrations'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', params: t.type({ query: t.intersection([ - dataStreamTypesRt, + typeRt, t.partial({ datasetQuery: t.string, }), @@ -41,7 +40,6 @@ const statsRoute = createDatasetQualityServerRoute({ const fleetPluginStart = await plugins.fleet.start(); const packageClient = fleetPluginStart.packageService.asInternalUser; - const packages = await packageClient.getPackages(); const [dataStreams, dataStreamsStats] = await Promise.all([ getDataStreams({ @@ -52,22 +50,11 @@ const statsRoute = createDatasetQualityServerRoute({ getDataStreamsStats({ esClient, ...params.query }), ]); - const installedPackages = dataStreams.items.map((item) => item.integration); - - const integrations = packages - .filter((pkg) => installedPackages.includes(pkg.name)) - .map((p) => ({ - name: p.name, - title: p.title, - version: p.version, - icons: p.icons, - })); - return { dataStreamsStats: values( merge(keyBy(dataStreams.items, 'name'), keyBy(dataStreamsStats.items, 'name')) ), - integrations, + integrations: await getIntegrations({ packageClient, dataStreams: dataStreams.items }), }; }, }); @@ -77,7 +64,7 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ params: t.type({ query: t.intersection([ rangeRt, - dataStreamTypesRt, + typeRt, t.partial({ datasetQuery: t.string, }), diff --git a/x-pack/plugins/dataset_quality/server/types/data_stream.ts b/x-pack/plugins/dataset_quality/server/types/data_stream.ts deleted file mode 100644 index 1ccac8199a8b6..0000000000000 --- a/x-pack/plugins/dataset_quality/server/types/data_stream.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ByteSize } from '@elastic/elasticsearch/lib/api/types'; -import { Integration } from './integration'; -export interface DataStreamStat { - name: string; - size?: ByteSize; - size_bytes?: number; - last_activity?: number; - integration?: Integration; -} - -export type DataStreamTypes = 'logs' | 'metrics' | 'traces' | 'synthetics' | 'profiling'; diff --git a/x-pack/plugins/dataset_quality/server/types/default_api_types.ts b/x-pack/plugins/dataset_quality/server/types/default_api_types.ts index e36bb1e58f65a..6148832dad140 100644 --- a/x-pack/plugins/dataset_quality/server/types/default_api_types.ts +++ b/x-pack/plugins/dataset_quality/server/types/default_api_types.ts @@ -8,16 +8,21 @@ import * as t from 'io-ts'; import { isoToEpochRt } from '@kbn/io-ts-utils'; -export const dataStreamTypesRt = t.partial({ - type: t.union([ - t.literal('logs'), - t.literal('metrics'), - t.literal('traces'), - t.literal('synthetics'), - t.literal('profiling'), - ]), +// https://github.com/gcanti/io-ts/blob/master/index.md#union-of-string-literals +export const dataStreamTypesRt = t.keyof({ + logs: null, + metrics: null, + traces: null, + synthetics: null, + profiling: null, }); +export const typeRt = t.partial({ + type: dataStreamTypesRt, +}); + +export type DataStreamTypes = t.TypeOf; + export const rangeRt = t.type({ start: isoToEpochRt, end: isoToEpochRt, diff --git a/x-pack/plugins/dataset_quality/server/types/integration.ts b/x-pack/plugins/dataset_quality/server/types/integration.ts deleted file mode 100644 index 2595a120c8b70..0000000000000 --- a/x-pack/plugins/dataset_quality/server/types/integration.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface Integration { - name: string; - title?: string; - version?: string; - icons?: IntegrationIcon[]; -} - -export interface IntegrationIcon { - path: string; - src: string; - title?: string; - size?: string; - type?: string; -} diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts new file mode 100644 index 0000000000000..ba38cc432ff8c --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const synthtrace = getService('logSynthtraceEsClient'); + const datasetQualityApiClient = getService('datasetQualityApiClient'); + const start = '2023-12-11T18:00:00.000Z'; + const end = '2023-12-11T18:01:00.000Z'; + + async function callApiAs(user: DatasetQualityApiClientKey) { + return await datasetQualityApiClient[user]({ + endpoint: 'GET /internal/dataset_quality/data_streams/degraded_docs', + params: { + query: { + type: 'logs', + start, + end, + }, + }, + }); + } + + registry.when('Degraded docs', { config: 'basic' }, () => { + describe('and there are log documents', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset('synth.1') + .defaults({ + 'log.file.path': '/my-service.log', + }) + ), + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset('synth.2') + .logLevel( + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?' + ) + .defaults({ + 'log.file.path': '/my-service.log', + }) + ), + ]); + }); + + it('returns stats correctly', async () => { + const stats = await callApiAs('datasetQualityLogsUser'); + expect(stats.body.degradedDocs.length).to.be(2); + + const percentages = stats.body.degradedDocs.reduce( + (acc, curr) => ({ + ...acc, + [curr.dataset]: curr.percentage, + }), + {} as Record + ); + + expect(percentages['logs-synth.1-default']).to.be(0); + expect(percentages['logs-synth.2-default']).to.be(100); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); + + describe('and there are not log documents', () => { + it('returns stats correctly', async () => { + const stats = await callApiAs('datasetQualityLogsUser'); + + expect(stats.body.degradedDocs.length).to.be(0); + }); + }); + }); +} diff --git a/x-pack/test/dataset_quality_api_integration/tests/es_utils.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts similarity index 100% rename from x-pack/test/dataset_quality_api_integration/tests/es_utils.ts rename to x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts similarity index 86% rename from x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts rename to x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts index 6d11326bf213e..7fa60dcb4b118 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts @@ -7,10 +7,10 @@ import { log, timerange } from '@kbn/apm-synthtrace-client'; import expect from '@kbn/expect'; -import { DatasetQualityApiClientKey } from '../common/config'; -import { DatasetQualityApiError } from '../common/dataset_quality_api_supertest'; -import { FtrProviderContext } from '../common/ftr_provider_context'; -import { expectToReject } from '../utils'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { expectToReject } from '../../utils'; import { cleanLogIndexTemplate, addIntegrationToLogIndexTemplate } from './es_utils'; export default function ApiTest({ getService }: FtrProviderContext) { @@ -43,7 +43,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when required privileges are set', () => { - describe('and uncategorized datastreams', () => { + describe('and categorized datastreams', () => { const integration = 'my-custom-integration'; before(async () => { @@ -67,8 +67,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(stats.body.dataStreamsStats.length).to.be(1); expect(stats.body.dataStreamsStats[0].integration).to.be(integration); expect(stats.body.dataStreamsStats[0].size).not.empty(); - expect(stats.body.dataStreamsStats[0].size_bytes).greaterThan(0); - expect(stats.body.dataStreamsStats[0].last_activity).greaterThan(0); + expect(stats.body.dataStreamsStats[0].sizeBytes).greaterThan(0); + expect(stats.body.dataStreamsStats[0].lastActivity).greaterThan(0); }); after(async () => { @@ -77,7 +77,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); - describe('and categorized datastreams', () => { + describe('and uncategorized datastreams', () => { before(async () => { await synthtrace.index([ timerange('2023-11-20T15:00:00.000Z', '2023-11-20T15:01:00.000Z') @@ -97,8 +97,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(stats.body.dataStreamsStats.length).to.be(1); expect(stats.body.dataStreamsStats[0].integration).not.ok(); expect(stats.body.dataStreamsStats[0].size).not.empty(); - expect(stats.body.dataStreamsStats[0].size_bytes).greaterThan(0); - expect(stats.body.dataStreamsStats[0].last_activity).greaterThan(0); + expect(stats.body.dataStreamsStats[0].sizeBytes).greaterThan(0); + expect(stats.body.dataStreamsStats[0].lastActivity).greaterThan(0); }); after(async () => {