[Dataset quality] Including integration information within stats (ela…

…stic#171663) ### Changes - `sortOrder` was removed as a param - `GET /internal/dataset_quality/data_streams/stats` now includes information related to integrations (name, title, version, icons) Response now looks like ``` { "items": [ { "name": "logs-elastic_agent-default", "size": "1.5mb", "size_bytes": 1645068, "last_activity": 1700378618104 }, { "name": "logs-elastic_agent.filebeat-default", "size": "3.6mb", "size_bytes": 3860281, "last_activity": 1700472909107 }, { "name": "logs-elastic_agent.metricbeat-default", "size": "1.6mb", "size_bytes": 1773572, "last_activity": 1700142440834 }, { "name": "logs-system.auth-default", "integration": "system", "size": "6.8mb", "size_bytes": 7226358, "last_activity": 1700472604014 }, { "name": "logs-system.syslog-default", "integration": "system", "size": "1.2gb", "size_bytes": 1302089724, "last_activity": 1700473867000 } ], "integrations": [ { "name": "system", "title": "System", "version": "1.49.0", "icons": [ { "src": "/img/system.svg", "path": "/package/system/1.49.0/img/system.svg", "title": "system", "size": "1000x1000", "type": "image/svg+xml" } ] } ] } ```
rshen91 · Nov 22, 2023 · 74dea1e · 74dea1e
1 parent 1bc5b6f
commit 74dea1e
Show file tree

Hide file tree

Showing 12 changed files with 83 additions and 124 deletions.
diff --git a/x-pack/plugins/dataset_quality/kibana.jsonc b/x-pack/plugins/dataset_quality/kibana.jsonc
@@ -8,7 +8,7 @@
     "server": true,
     "browser": true,
     "configPath": ["xpack", "datasetQuality"],
-    "requiredPlugins": ["data", "kibanaReact", "kibanaUtils", "controls", "embeddable", "share"],
+    "requiredPlugins": ["data", "kibanaReact", "kibanaUtils", "controls", "embeddable", "share", "fleet"],
     "optionalPlugins": [],
     "requiredBundles": [],
     "extraPublicDirs": ["common"]

diff --git a/...gins/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts b/...gins/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts
@@ -114,7 +114,6 @@ describe('getDataStreams', () => {
       esClient: esClientMock,
       type: 'logs',
       datasetQuery: 'nginx',
-      sortOrder: 'asc',
       uncategorisedOnly: true,
     });
     expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith(expect.anything(), {
@@ -129,7 +128,6 @@ describe('getDataStreams', () => {
         esClient: esClientMock,
         type: 'logs',
         datasetQuery: 'nginx',
-        sortOrder: 'asc',
         uncategorisedOnly: true,
       });
       expect(results.items.length).toBe(1);
@@ -140,62 +138,36 @@ describe('getDataStreams', () => {
         esClient: esClientMock,
         type: 'logs',
         datasetQuery: 'nginx',
-        sortOrder: 'asc',
         uncategorisedOnly: false,
       });
       expect(results.items.length).toBe(5);
     });
   });
-  describe('Can be sorted', () => {
-    it('Ascending', async () => {
-      const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
-      const results = await getDataStreams({
-        esClient: esClientMock,
-        type: 'logs',
-        datasetQuery: 'nginx',
-        sortOrder: 'asc',
-        uncategorisedOnly: false,
-      });
-      expect(results.items[0].name).toBe('logs-elastic_agent-default');
-    });
-    it('Descending', async () => {
-      const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
-      const results = await getDataStreams({
-        esClient: esClientMock,
-        type: 'logs',
-        datasetQuery: 'nginx',
-        sortOrder: 'desc',
-        uncategorisedOnly: false,
-      });
-      expect(results.items[0].name).toBe('logs-test.test-default');
-    });
-  });
   it('Formats the items correctly', async () => {
     const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
     const results = await getDataStreams({
       esClient: esClientMock,
       type: 'logs',
-      sortOrder: 'desc',
       uncategorisedOnly: false,
     });
-    expect(results.items).toEqual([
-      { name: 'logs-test.test-default' },
+    expect(results.items.sort()).toEqual([
       {
-        name: 'logs-elastic_agent.metricbeat-default',
-        integration: { name: 'elastic_agent', managed_by: 'fleet' },
+        name: 'logs-elastic_agent-default',
+        integration: 'elastic_agent',
       },
       {
-        name: 'logs-elastic_agent.fleet_server-default',
-        integration: { name: 'elastic_agent', managed_by: 'fleet' },
+        name: 'logs-elastic_agent.filebeat-default',
+        integration: 'elastic_agent',
       },
       {
-        name: 'logs-elastic_agent.filebeat-default',
-        integration: { name: 'elastic_agent', managed_by: 'fleet' },
+        name: 'logs-elastic_agent.fleet_server-default',
+        integration: 'elastic_agent',
       },
       {
-        name: 'logs-elastic_agent-default',
-        integration: { name: 'elastic_agent', managed_by: 'fleet' },
+        name: 'logs-elastic_agent.metricbeat-default',
+        integration: 'elastic_agent',
       },
+      { name: 'logs-test.test-default' },
     ]);
   });
 });
diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts
@@ -13,10 +13,9 @@ export async function getDataStreams(options: {
   esClient: ElasticsearchClient;
   type?: DataStreamTypes;
   datasetQuery?: string;
-  sortOrder: 'asc' | 'desc';
   uncategorisedOnly: boolean;
 }) {
-  const { esClient, type, datasetQuery, uncategorisedOnly, sortOrder } = options;
+  const { esClient, type, datasetQuery, uncategorisedOnly } = options;
 
   const allDataStreams = await dataStreamService.getMatchingDataStreams(esClient, {
     type: type ?? '*',
@@ -31,25 +30,10 @@ export async function getDataStreams(options: {
 
   const mappedDataStreams = filteredDataStreams.map((dataStream) => ({
     name: dataStream.name,
-    ...(dataStream._meta
-      ? {
-          integration: {
-            name: dataStream._meta?.package?.name,
-            managed_by: dataStream._meta?.managed_by,
-          },
-        }
-      : {}),
+    integration: dataStream._meta?.package?.name,
   }));
 
-  const sortedDataStreams = mappedDataStreams.sort((a, b) => {
-    if (sortOrder === 'desc') {
-      return b.name.localeCompare(a.name);
-    }
-
-    return a.name.localeCompare(b.name);
-  });
-
   return {
-    items: sortedDataStreams,
+    items: mappedDataStreams,
   };
 }
diff --git a/..._quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts b/..._quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts
@@ -63,52 +63,30 @@ describe('getDataStreams', () => {
       esClient: esClientMock,
       type: 'logs',
       datasetQuery: 'nginx',
-      sortOrder: 'asc',
     });
     expect(dataStreamService.getMatchingDataStreamsStats).toHaveBeenCalledWith(expect.anything(), {
       type: 'logs',
       dataset: '*nginx*',
     });
   });
-  describe('Can be sorted', () => {
-    it('Ascending', async () => {
-      const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
-      const results = await getDataStreamsStats({
-        esClient: esClientMock,
-        type: 'logs',
-        sortOrder: 'asc',
-      });
-      expect(results.items[0].name).toBe('logs-elastic_agent-default');
-    });
-    it('Descending', async () => {
-      const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
-      const results = await getDataStreamsStats({
-        esClient: esClientMock,
-        type: 'logs',
-        sortOrder: 'desc',
-      });
-      expect(results.items[0].name).toBe('logs-test.test-default');
-    });
-  });
   it('Formats the items correctly', async () => {
     const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
     const results = await getDataStreamsStats({
       esClient: esClientMock,
       type: 'logs',
-      sortOrder: 'desc',
     });
-    expect(results.items).toEqual([
+    expect(results.items.sort()).toEqual([
       {
-        name: 'logs-test.test-default',
-        size: '6.2mb',
-        size_bytes: 6570447,
-        last_activity: 1698913802000,
+        name: 'logs-elastic_agent-default',
+        size: '1gb',
+        size_bytes: 1170805528,
+        last_activity: 1698916071000,
       },
       {
-        name: 'logs-elastic_agent.metricbeat-default',
-        size: '1.6mb',
-        size_bytes: 1704807,
-        last_activity: 1698672046707,
+        name: 'logs-elastic_agent.filebeat-default',
+        size: '1.3mb',
+        size_bytes: 1459100,
+        last_activity: 1698902209996,
       },
       {
         name: 'logs-elastic_agent.fleet_server-default',
@@ -117,16 +95,16 @@ describe('getDataStreams', () => {
         last_activity: 1698914110010,
       },
       {
-        name: 'logs-elastic_agent.filebeat-default',
-        size: '1.3mb',
-        size_bytes: 1459100,
-        last_activity: 1698902209996,
+        name: 'logs-elastic_agent.metricbeat-default',
+        size: '1.6mb',
+        size_bytes: 1704807,
+        last_activity: 1698672046707,
       },
       {
-        name: 'logs-elastic_agent-default',
-        size: '1gb',
-        size_bytes: 1170805528,
-        last_activity: 1698916071000,
+        name: 'logs-test.test-default',
+        size: '6.2mb',
+        size_bytes: 6570447,
+        last_activity: 1698913802000,
       },
     ]);
   });

diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts
@@ -13,9 +13,8 @@ export async function getDataStreamsStats(options: {
   esClient: ElasticsearchClient;
   type?: DataStreamTypes;
   datasetQuery?: string;
-  sortOrder: 'asc' | 'desc';
 }) {
-  const { esClient, type, datasetQuery, sortOrder } = options;
+  const { esClient, type, datasetQuery } = options;
 
   const matchingDataStreamsStats = await dataStreamService.getMatchingDataStreamsStats(esClient, {
     type: type ?? '*',
@@ -31,15 +30,7 @@ export async function getDataStreamsStats(options: {
     };
   });
 
-  const sortedDataStreams = mappedDataStreams.sort((a, b) => {
-    if (sortOrder === 'desc') {
-      return b.name.localeCompare(a.name);
-    }
-
-    return a.name.localeCompare(b.name);
-  });
-
   return {
-    items: sortedDataStreams,
+    items: mappedDataStreams,
   };
 }
diff --git a/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts
@@ -7,7 +7,7 @@
 
 import * as t from 'io-ts';
 import { keyBy, merge, values } from 'lodash';
-import { dataStreamTypesRt, sortOrderRt } from '../../types/api_types';
+import { dataStreamTypesRt } from '../../types/api_types';
 import { DataStreamsStatResponse } from '../../types/data_stream';
 import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
 import { getDataStreams } from './get_data_streams';
@@ -21,19 +21,22 @@ const statsRoute = createDatasetQualityServerRoute({
       t.partial({
         datasetQuery: t.string,
       }),
-      sortOrderRt,
     ]),
   }),
   options: {
     tags: [],
   },
   async handler(resources): Promise<DataStreamsStatResponse> {
-    const { context, params } = resources;
+    const { context, params, plugins } = resources;
     const coreContext = await context.core;
 
     // Query datastreams as the current user as the Kibana internal user may not have all the required permissions
     const esClient = coreContext.elasticsearch.client.asCurrentUser;
 
+    const fleetPluginStart = await plugins.fleet.start();
+    const packageClient = fleetPluginStart.packageService.asInternalUser;
+    const packages = await packageClient.getPackages();
+
     const [dataStreams, dataStreamsStats] = await Promise.all([
       getDataStreams({
         esClient,
@@ -43,8 +46,20 @@ const statsRoute = createDatasetQualityServerRoute({
       getDataStreamsStats({ esClient, ...params.query }),
     ]);
 
+    const installedPackages = dataStreams.items.map((item) => item.integration);
+
+    const integrations = packages
+      .filter((pkg) => installedPackages.includes(pkg.name))
+      .map((p) => ({
+        name: p.name,
+        title: p.title,
+        version: p.version,
+        icons: p.icons,
+      }));
+
     return {
       items: values(merge(keyBy(dataStreams.items, 'name'), keyBy(dataStreamsStats.items, 'name'))),
+      integrations,
     };
   },
 });

diff --git a/x-pack/plugins/dataset_quality/server/types.ts b/x-pack/plugins/dataset_quality/server/types.ts
@@ -6,12 +6,15 @@
  */
 
 import { CustomRequestHandlerContext } from '@kbn/core/server';
+import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface DatasetQualityPluginSetupDependencies {}
+export interface DatasetQualityPluginSetupDependencies {
+  fleet: FleetSetupContract;
+}
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface DatasetQualityPluginStartDependencies {}
+export interface DatasetQualityPluginStartDependencies {
+  fleet: FleetStartContract;
+}
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface DatasetQualityPluginSetup {}

diff --git a/x-pack/plugins/dataset_quality/server/types/api_types.ts b/x-pack/plugins/dataset_quality/server/types/api_types.ts
@@ -16,7 +16,3 @@ export const dataStreamTypesRt = t.partial({
     t.literal('profiling'),
   ]),
 });
-
-export const sortOrderRt = t.type({
-  sortOrder: t.union([t.literal('asc'), t.literal('desc')]),
-});
diff --git a/x-pack/plugins/dataset_quality/server/types/data_stream.ts b/x-pack/plugins/dataset_quality/server/types/data_stream.ts
@@ -6,20 +6,19 @@
  */
 
 import { ByteSize } from '@elastic/elasticsearch/lib/api/types';
+import { Integration } from './integration';
 
 export interface DataStreamsStatResponse {
   items: DataStreamStat[];
+  integrations: Integration[];
 }
 
 export interface DataStreamStat {
   name: string;
   size?: ByteSize;
   size_bytes?: number;
   last_activity?: number;
-  integration?: {
-    name?: string;
-    managed_by?: string;
-  };
+  integration?: Integration;
 }
 
 export type DataStreamTypes = 'logs' | 'metrics' | 'traces' | 'synthetics' | 'profiling';
diff --git a/x-pack/plugins/dataset_quality/server/types/integration.ts b/x-pack/plugins/dataset_quality/server/types/integration.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface Integration {
+  name: string;
+  title?: string;
+  version?: string;
+  icons?: IntegrationIcon[];
+}
+
+export interface IntegrationIcon {
+  path: string;
+  src: string;
+  title?: string;
+  size?: string;
+  type?: string;
+}
diff --git a/x-pack/plugins/dataset_quality/tsconfig.json b/x-pack/plugins/dataset_quality/tsconfig.json
@@ -15,6 +15,7 @@
     "@kbn/core-plugins-server",
     "@kbn/core-elasticsearch-server-mocks",
     "@kbn/std",
+    "@kbn/fleet-plugin",
   ],
   "exclude": [
     "target/**/*",

diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts
@@ -21,7 +21,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       params: {
         query: {
           type: 'logs',
-          sortOrder: 'asc',
         },
       },
     });