Create usage collectors for connectors (#172280)

## Closes elastic/search-team#6297 ## Summary This PR adds usage collectors for connectors, to collect basic telemetry metrics, in plugin `enterprise_search` and `serverless_search`. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <[email protected]> (cherry picked from commit f5c78b9)
elastic · Dec 7, 2023 · 4f65cb6 · 4f65cb6
1 parent ab3a7b2
commit 4f65cb6
Show file tree

Hide file tree

Showing 10 changed files with 361 additions and 2 deletions.
diff --git a/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts b/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createCollectorFetchContextMock } from '@kbn/usage-collection-plugin/server/mocks';
+
+import { registerTelemetryUsageCollector } from './telemetry';
+
+describe('Connectors Telemetry Usage Collector', () => {
+  const makeUsageCollectorStub = jest.fn();
+  const registerStub = jest.fn();
+  const usageCollectionMock = {
+    makeUsageCollector: makeUsageCollectorStub,
+    registerCollector: registerStub,
+  } as any;
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('registerTelemetryUsageCollector', () => {
+    it('should make and register the usage collector', () => {
+      registerTelemetryUsageCollector(usageCollectionMock);
+
+      expect(registerStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('connectors');
+      expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true);
+    });
+  });
+
+  describe('fetchTelemetryMetrics', () => {
+    it('should return telemetry data', async () => {
+      const fetchContextMock = createCollectorFetchContextMock();
+      fetchContextMock.esClient.count = jest.fn().mockImplementation((query: any) =>
+        Promise.resolve({
+          count: query.query.bool.filter[0].term.is_native ? 5 : 2,
+        })
+      );
+      registerTelemetryUsageCollector(usageCollectionMock);
+      const telemetryMetrics = await makeUsageCollectorStub.mock.calls[0][0].fetch(
+        fetchContextMock
+      );
+
+      expect(telemetryMetrics).toEqual({
+        native: {
+          total: 5,
+        },
+        clients: {
+          total: 2,
+        },
+      });
+    });
+  });
+});
diff --git a/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts b/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+import { CONNECTORS_INDEX } from '@kbn/search-connectors';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+
+interface Telemetry {
+  native: {
+    total: number;
+  };
+  clients: {
+    total: number;
+  };
+}
+
+/**
+ * Register the telemetry collector
+ */
+
+export const registerTelemetryUsageCollector = (usageCollection: UsageCollectionSetup) => {
+  const telemetryUsageCollector = usageCollection.makeUsageCollector<Telemetry>({
+    type: 'connectors',
+    isReady: () => true,
+    schema: {
+      native: {
+        total: { type: 'long' },
+      },
+      clients: {
+        total: { type: 'long' },
+      },
+    },
+    async fetch({ esClient }) {
+      return await fetchTelemetryMetrics(esClient);
+    },
+  });
+  usageCollection.registerCollector(telemetryUsageCollector);
+};
+
+/**
+ * Fetch the aggregated telemetry metrics
+ */
+
+export const fetchTelemetryMetrics = async (client: ElasticsearchClient): Promise<Telemetry> => {
+  const [nativeCountResponse, clientsCountResponse] = await Promise.all([
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: true,
+              },
+            },
+          ],
+          must_not: [
+            {
+              term: {
+                service_type: {
+                  value: 'elastic-crawler',
+                },
+              },
+            },
+          ],
+        },
+      },
+    }),
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: false,
+              },
+            },
+          ],
+        },
+      },
+    }),
+  ]);
+
+  return {
+    native: {
+      total: nativeCountResponse.count,
+    },
+    clients: {
+      total: clientsCountResponse.count,
+    },
+  } as Telemetry;
+};
diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts
@@ -50,6 +50,7 @@ import {
 } from '../common/guided_onboarding/search_guide_config';
 
 import { registerTelemetryUsageCollector as registerASTelemetryUsageCollector } from './collectors/app_search/telemetry';
+import { registerTelemetryUsageCollector as registerCNTelemetryUsageCollector } from './collectors/connectors/telemetry';
 import { registerTelemetryUsageCollector as registerESTelemetryUsageCollector } from './collectors/enterprise_search/telemetry';
 import { registerTelemetryUsageCollector as registerWSTelemetryUsageCollector } from './collectors/workplace_search/telemetry';
 import { registerEnterpriseSearchIntegrations } from './integrations';
@@ -262,6 +263,7 @@ export class EnterpriseSearchPlugin implements Plugin {
 
       if (usageCollection) {
         registerESTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger);
+        registerCNTelemetryUsageCollector(usageCollection);
         if (config.canDeployEntSearch) {
           registerASTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger);
           registerWSTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger);

diff --git a/x-pack/plugins/serverless_search/kibana.jsonc b/x-pack/plugins/serverless_search/kibana.jsonc
@@ -28,6 +28,7 @@
     ],
     "optionalPlugins": [
       "indexManagement",
+      "usageCollection",
     ],
     "requiredBundles": [
       "kibanaReact"

diff --git a/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts b/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { registerTelemetryUsageCollector } from './telemetry';
+import { createCollectorFetchContextMock } from '@kbn/usage-collection-plugin/server/mocks';
+
+describe('Connectors Serverless Telemetry Usage Collector', () => {
+  const makeUsageCollectorStub = jest.fn();
+  const registerStub = jest.fn();
+  const usageCollectionMock = {
+    makeUsageCollector: makeUsageCollectorStub,
+    registerCollector: registerStub,
+  } as any;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('registerTelemetryUsageCollector', () => {
+    it('should make and register the usage collector', () => {
+      registerTelemetryUsageCollector(usageCollectionMock);
+
+      expect(registerStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('connectors_serverless');
+      expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true);
+    });
+  });
+
+  describe('fetchTelemetryMetrics', () => {
+    it('should return telemetry data', async () => {
+      const fetchContextMock = createCollectorFetchContextMock();
+      fetchContextMock.esClient.count = jest.fn().mockImplementation((query: any) =>
+        Promise.resolve({
+          count: query.query.bool.filter[0].term.is_native ? 5 : 2,
+        })
+      );
+      registerTelemetryUsageCollector(usageCollectionMock);
+      const telemetryMetrics = await makeUsageCollectorStub.mock.calls[0][0].fetch(
+        fetchContextMock
+      );
+
+      expect(telemetryMetrics).toEqual({
+        native: {
+          total: 5,
+        },
+        clients: {
+          total: 2,
+        },
+      });
+    });
+  });
+});
diff --git a/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts b/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+import { CONNECTORS_INDEX } from '@kbn/search-connectors';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+
+interface Telemetry {
+  native: {
+    total: number;
+  };
+  clients: {
+    total: number;
+  };
+}
+
+/**
+ * Register the telemetry collector
+ */
+
+export const registerTelemetryUsageCollector = (usageCollection: UsageCollectionSetup) => {
+  const telemetryUsageCollector = usageCollection.makeUsageCollector<Telemetry>({
+    type: 'connectors_serverless',
+    isReady: () => true,
+    schema: {
+      native: {
+        total: { type: 'long' },
+      },
+      clients: {
+        total: { type: 'long' },
+      },
+    },
+    async fetch({ esClient }) {
+      return await fetchTelemetryMetrics(esClient);
+    },
+  });
+  usageCollection.registerCollector(telemetryUsageCollector);
+};
+
+/**
+ * Fetch the aggregated telemetry metrics
+ */
+
+export const fetchTelemetryMetrics = async (client: ElasticsearchClient): Promise<Telemetry> => {
+  const [nativeCountResponse, clientsCountResponse] = await Promise.all([
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: true,
+              },
+            },
+          ],
+          must_not: [
+            {
+              term: {
+                service_type: {
+                  value: 'elastic-crawler',
+                },
+              },
+            },
+          ],
+        },
+      },
+    }),
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: false,
+              },
+            },
+          ],
+        },
+      },
+    }),
+  ]);
+
+  return {
+    native: {
+      total: nativeCountResponse.count,
+    },
+    clients: {
+      total: clientsCountResponse.count,
+    },
+  } as Telemetry;
+};
diff --git a/x-pack/plugins/serverless_search/server/plugin.ts b/x-pack/plugins/serverless_search/server/plugin.ts
@@ -27,6 +27,7 @@ import type {
   StartDependencies,
 } from './types';
 import { registerConnectorsRoutes } from './routes/connectors_routes';
+import { registerTelemetryUsageCollector } from './collectors/connectors/telemetry';
 
 export interface RouteDependencies {
   http: CoreSetup<StartDependencies>['http'];
@@ -77,7 +78,7 @@ export class ServerlessSearchPlugin
 
   public setup(
     { getStartServices, http }: CoreSetup<StartDependencies>,
-    pluginsSetup: SetupDependencies
+    { serverless, usageCollection }: SetupDependencies
   ) {
     const router = http.createRouter();
     getStartServices().then(([, { security }]) => {
@@ -94,7 +95,13 @@ export class ServerlessSearchPlugin
       registerIndicesRoutes(dependencies);
     });
 
-    pluginsSetup.serverless.setupProjectSettings(SEARCH_PROJECT_SETTINGS);
+    if (usageCollection) {
+      getStartServices().then(() => {
+        registerTelemetryUsageCollector(usageCollection);
+      });
+    }
+
+    serverless.setupProjectSettings(SEARCH_PROJECT_SETTINGS);
     return {};
   }
 

diff --git a/x-pack/plugins/serverless_search/server/types.ts b/x-pack/plugins/serverless_search/server/types.ts
@@ -8,6 +8,7 @@
 import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
 import type { SecurityPluginStart } from '@kbn/security-plugin/server';
 import type { ServerlessPluginSetup } from '@kbn/serverless/server';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface ServerlessSearchPluginSetup {}
@@ -20,4 +21,5 @@ export interface StartDependencies {
 }
 export interface SetupDependencies {
   serverless: ServerlessPluginSetup;
+  usageCollection?: UsageCollectionSetup;
 }
diff --git a/x-pack/plugins/serverless_search/tsconfig.json b/x-pack/plugins/serverless_search/tsconfig.json
@@ -38,5 +38,6 @@
     "@kbn/data-views-plugin",
     "@kbn/kibana-utils-plugin",
     "@kbn/index-management-plugin",
+    "@kbn/usage-collection-plugin",
   ]
 }