[Dataset quality] Dedicated column for namespace (elastic#173087)

## Changes 1. Splits `datasetname + namespace` from Dataset name column. 2. Creates a dedicated column for `namespace`. 3. Bring the human readable name from datasets. #### Before <img width="2411" alt="image" src="https://github.com/elastic/kibana/assets/1313018/0df47bb9-3109-4670-af4a-e917bc75deb1"> #### After <img width="2414" alt="image" src="https://github.com/elastic/kibana/assets/1313018/4f2c9db7-b8b4-4520-89b2-41ba0326524a">
rshen91 · Dec 18, 2023 · 9a65337 · 9a65337
1 parent e3ef0c8
commit 9a65337
Show file tree

Hide file tree

Showing 15 changed files with 224 additions and 99 deletions.
diff --git a/x-pack/plugins/dataset_quality/common/api_types.ts b/x-pack/plugins/dataset_quality/common/api_types.ts
@@ -7,7 +7,7 @@
 
 import * as rt from 'io-ts';
 
-export const datasetStatRt = rt.intersection([
+export const dataStreamStatRt = rt.intersection([
   rt.type({
     name: rt.string,
   }),
@@ -19,6 +19,8 @@ export const datasetStatRt = rt.intersection([
   }),
 ]);
 
+export type DataStreamStat = rt.TypeOf<typeof dataStreamStatRt>;
+
 export const integrationIconRt = rt.intersection([
   rt.type({
     path: rt.string,
@@ -39,9 +41,12 @@ export const integrationRt = rt.intersection([
     title: rt.string,
     version: rt.string,
     icons: rt.array(integrationIconRt),
+    datasets: rt.record(rt.string, rt.string),
   }),
 ]);
 
+export type Integration = rt.TypeOf<typeof integrationRt>;
+
 export const degradedDocsRt = rt.type({
   dataset: rt.string,
   percentage: rt.number,
@@ -52,7 +57,7 @@ export type DegradedDocs = rt.TypeOf<typeof degradedDocsRt>;
 export const getDataStreamsStatsResponseRt = rt.exact(
   rt.intersection([
     rt.type({
-      dataStreamsStats: rt.array(datasetStatRt),
+      dataStreamsStats: rt.array(dataStreamStatRt),
     }),
     rt.type({
       integrations: rt.array(integrationRt),

diff --git a/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts
@@ -10,16 +10,18 @@ import { DataStreamStatType, IntegrationType } from './types';
 
 export class DataStreamStat {
   name: DataStreamStatType['name'];
+  namespace: string;
   title: string;
   size?: DataStreamStatType['size'];
-  sizeBytes?: DataStreamStatType['size_bytes'];
-  lastActivity?: DataStreamStatType['last_activity'];
+  sizeBytes?: DataStreamStatType['sizeBytes'];
+  lastActivity?: DataStreamStatType['lastActivity'];
   integration?: IntegrationType;
   degradedDocs?: number;
 
   private constructor(dataStreamStat: DataStreamStat) {
     this.name = dataStreamStat.name;
     this.title = dataStreamStat.title ?? dataStreamStat.name;
+    this.namespace = dataStreamStat.namespace;
     this.size = dataStreamStat.size;
     this.sizeBytes = dataStreamStat.sizeBytes;
     this.lastActivity = dataStreamStat.lastActivity;
@@ -32,10 +34,11 @@ export class DataStreamStat {
 
     const dataStreamStatProps = {
       name: dataStreamStat.name,
-      title: `${dataset}-${namespace}`,
+      title: dataStreamStat.integration?.datasets?.[dataset] ?? dataset,
+      namespace,
       size: dataStreamStat.size,
-      sizeBytes: dataStreamStat.size_bytes,
-      lastActivity: dataStreamStat.last_activity,
+      sizeBytes: dataStreamStat.sizeBytes,
+      lastActivity: dataStreamStat.lastActivity,
       integration: dataStreamStat.integration
         ? Integration.create(dataStreamStat.integration)
         : undefined,

diff --git a/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx b/x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx
@@ -7,6 +7,7 @@
 
 import React from 'react';
 import {
+  EuiBadge,
   EuiBasicTableColumn,
   EuiCode,
   EuiFlexGroup,
@@ -33,6 +34,10 @@ const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', {
   defaultMessage: 'Dataset Name',
 });
 
+const namespaceColumnName = i18n.translate('xpack.datasetQuality.namespaceColumnName', {
+  defaultMessage: 'Namespace',
+});
+
 const sizeColumnName = i18n.translate('xpack.datasetQuality.sizeColumnName', {
   defaultMessage: 'Size',
 });
@@ -122,6 +127,14 @@ export const getDatasetQualitTableColumns = ({
         );
       },
     },
+    {
+      name: namespaceColumnName,
+      field: 'namespace',
+      sortable: true,
+      render: (_, dataStreamStat: DataStreamStat) => (
+        <EuiBadge color="hollow">{dataStreamStat.namespace}</EuiBadge>
+      ),
+    },
     {
       name: sizeColumnName,
       field: 'size',

diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts
@@ -6,8 +6,8 @@
  */
 
 import type { ElasticsearchClient } from '@kbn/core/server';
+import { DataStreamTypes } from '../../../types/default_api_types';
 import { dataStreamService } from '../../../services';
-import { DataStreamTypes } from '../../../types/data_stream';
 
 export async function getDataStreams(options: {
   esClient: ElasticsearchClient;

diff --git a/..._quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts b/..._quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts
@@ -79,32 +79,32 @@ describe('getDataStreams', () => {
       {
         name: 'logs-elastic_agent-default',
         size: '1gb',
-        size_bytes: 1170805528,
-        last_activity: 1698916071000,
+        sizeBytes: 1170805528,
+        lastActivity: 1698916071000,
       },
       {
         name: 'logs-elastic_agent.filebeat-default',
         size: '1.3mb',
-        size_bytes: 1459100,
-        last_activity: 1698902209996,
+        sizeBytes: 1459100,
+        lastActivity: 1698902209996,
       },
       {
         name: 'logs-elastic_agent.fleet_server-default',
         size: '2.9mb',
-        size_bytes: 3052148,
-        last_activity: 1698914110010,
+        sizeBytes: 3052148,
+        lastActivity: 1698914110010,
       },
       {
         name: 'logs-elastic_agent.metricbeat-default',
         size: '1.6mb',
-        size_bytes: 1704807,
-        last_activity: 1698672046707,
+        sizeBytes: 1704807,
+        lastActivity: 1698672046707,
       },
       {
         name: 'logs-test.test-default',
         size: '6.2mb',
-        size_bytes: 6570447,
-        last_activity: 1698913802000,
+        sizeBytes: 6570447,
+        lastActivity: 1698913802000,
       },
     ]);
   });

diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts
@@ -6,8 +6,8 @@
  */
 
 import type { ElasticsearchClient } from '@kbn/core/server';
+import { DataStreamTypes } from '../../../types/default_api_types';
 import { dataStreamService } from '../../../services';
-import { DataStreamTypes } from '../../../types/data_stream';
 
 export async function getDataStreamsStats(options: {
   esClient: ElasticsearchClient;
@@ -24,9 +24,9 @@ export async function getDataStreamsStats(options: {
   const mappedDataStreams = matchingDataStreamsStats.map((dataStream) => {
     return {
       name: dataStream.data_stream,
-      size: dataStream.store_size,
-      size_bytes: dataStream.store_size_bytes,
-      last_activity: dataStream.maximum_timestamp,
+      size: dataStream.store_size?.toString(),
+      sizeBytes: dataStream.store_size_bytes,
+      lastActivity: dataStream.maximum_timestamp,
     };
   });
 

diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts
@@ -14,7 +14,7 @@ import {
   DATA_STREAM_TYPE,
   _IGNORED,
 } from '../../../common/es_fields';
-import { DataStreamTypes } from '../../types/data_stream';
+import { DataStreamTypes } from '../../types/default_api_types';
 import { createDatasetQualityESClient, wildcardQuery } from '../../utils';
 
 export async function getDegradedDocsPaginated(options: {

diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_integrations.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_integrations.ts
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PackageClient } from '@kbn/fleet-plugin/server';
+import { DataStreamStat, Integration } from '../../../common/api_types';
+
+export async function getIntegrations(options: {
+  packageClient: PackageClient;
+  dataStreams: DataStreamStat[];
+}): Promise<Integration[]> {
+  const { packageClient, dataStreams } = options;
+
+  const packages = await packageClient.getPackages();
+  const installedPackages = dataStreams.map((item) => item.integration);
+
+  return Promise.all(
+    packages
+      .filter((pkg) => installedPackages.includes(pkg.name))
+      .map(async (p) => ({
+        name: p.name,
+        title: p.title,
+        version: p.version,
+        icons: p.icons,
+        datasets: await getDatasets({
+          packageClient,
+          name: p.name,
+          version: p.version,
+        }),
+      }))
+  );
+}
+
+const getDatasets = async (options: {
+  packageClient: PackageClient;
+  name: string;
+  version: string;
+}) => {
+  const { packageClient, name, version } = options;
+
+  const pkg = await packageClient.getPackage(name, version);
+
+  return pkg.packageInfo.data_streams?.reduce(
+    (acc, curr) => ({
+      ...acc,
+      [curr.dataset]: curr.title,
+    }),
+    {}
+  );
+};
diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts
@@ -7,20 +7,19 @@
 
 import * as t from 'io-ts';
 import { keyBy, merge, values } from 'lodash';
-import { DataStreamStat } from '../../types/data_stream';
-import { dataStreamTypesRt, rangeRt } from '../../types/default_api_types';
-import { Integration } from '../../types/integration';
+import { typeRt, rangeRt } from '../../types/default_api_types';
 import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
 import { getDataStreams } from './get_data_streams';
 import { getDataStreamsStats } from './get_data_streams_stats';
 import { getDegradedDocsPaginated } from './get_degraded_docs';
-import { DegradedDocs } from '../../../common/api_types';
+import { DegradedDocs, DataStreamStat, Integration } from '../../../common/api_types';
+import { getIntegrations } from './get_integrations';
 
 const statsRoute = createDatasetQualityServerRoute({
   endpoint: 'GET /internal/dataset_quality/data_streams/stats',
   params: t.type({
     query: t.intersection([
-      dataStreamTypesRt,
+      typeRt,
       t.partial({
         datasetQuery: t.string,
       }),
@@ -41,7 +40,6 @@ const statsRoute = createDatasetQualityServerRoute({
 
     const fleetPluginStart = await plugins.fleet.start();
     const packageClient = fleetPluginStart.packageService.asInternalUser;
-    const packages = await packageClient.getPackages();
 
     const [dataStreams, dataStreamsStats] = await Promise.all([
       getDataStreams({
@@ -52,22 +50,11 @@ const statsRoute = createDatasetQualityServerRoute({
       getDataStreamsStats({ esClient, ...params.query }),
     ]);
 
-    const installedPackages = dataStreams.items.map((item) => item.integration);
-
-    const integrations = packages
-      .filter((pkg) => installedPackages.includes(pkg.name))
-      .map((p) => ({
-        name: p.name,
-        title: p.title,
-        version: p.version,
-        icons: p.icons,
-      }));
-
     return {
       dataStreamsStats: values(
         merge(keyBy(dataStreams.items, 'name'), keyBy(dataStreamsStats.items, 'name'))
       ),
-      integrations,
+      integrations: await getIntegrations({ packageClient, dataStreams: dataStreams.items }),
     };
   },
 });
@@ -77,7 +64,7 @@ const degradedDocsRoute = createDatasetQualityServerRoute({
   params: t.type({
     query: t.intersection([
       rangeRt,
-      dataStreamTypesRt,
+      typeRt,
       t.partial({
         datasetQuery: t.string,
       }),

diff --git a/x-pack/plugins/dataset_quality/server/types/data_stream.ts b/x-pack/plugins/dataset_quality/server/types/data_stream.ts
diff --git a/x-pack/plugins/dataset_quality/server/types/default_api_types.ts b/x-pack/plugins/dataset_quality/server/types/default_api_types.ts
@@ -8,16 +8,21 @@
 import * as t from 'io-ts';
 import { isoToEpochRt } from '@kbn/io-ts-utils';
 
-export const dataStreamTypesRt = t.partial({
-  type: t.union([
-    t.literal('logs'),
-    t.literal('metrics'),
-    t.literal('traces'),
-    t.literal('synthetics'),
-    t.literal('profiling'),
-  ]),
+// https://github.com/gcanti/io-ts/blob/master/index.md#union-of-string-literals
+export const dataStreamTypesRt = t.keyof({
+  logs: null,
+  metrics: null,
+  traces: null,
+  synthetics: null,
+  profiling: null,
 });
 
+export const typeRt = t.partial({
+  type: dataStreamTypesRt,
+});
+
+export type DataStreamTypes = t.TypeOf<typeof dataStreamTypesRt>;
+
 export const rangeRt = t.type({
   start: isoToEpochRt,
   end: isoToEpochRt,

diff --git a/x-pack/plugins/dataset_quality/server/types/integration.ts b/x-pack/plugins/dataset_quality/server/types/integration.ts