[8.12] Create usage collectors for connectors (elastic#172280) (elast…

…ic#172774)

# Backport

This will backport the following commits from `main` to `8.12`:
- [Create usage collectors for connectors
(elastic#172280)](elastic#172280)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Chenhui
Wang","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-12-07T07:04:18Z","message":"Create
usage collectors for connectors (elastic#172280)\n\n## Closes
https://github.com/elastic/enterprise-search-team/issues/6297\r\n\r\n##
Summary\r\n\r\nThis PR adds usage collectors for connectors, to collect
basic telemetry\r\nmetrics, in plugin `enterprise_search` and
`serverless_search`.\r\n\r\n\r\n### Checklist\r\n\r\nDelete any items
that are not applicable to this PR.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n\r\n\r\n### For
maintainers\r\n\r\n- [ ] This was checked for breaking API changes and
was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"f5c78b966b32ed3077b7795308f8902d1120013f","branchLabelMapping":{"^v8.13.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","auto-backport","v8.12.0"],"number":172280,"url":"https://github.com/elastic/kibana/pull/172280","mergeCommit":{"message":"Create
usage collectors for connectors (elastic#172280)\n\n## Closes
https://github.com/elastic/enterprise-search-team/issues/6297\r\n\r\n##
Summary\r\n\r\nThis PR adds usage collectors for connectors, to collect
basic telemetry\r\nmetrics, in plugin `enterprise_search` and
`serverless_search`.\r\n\r\n\r\n### Checklist\r\n\r\nDelete any items
that are not applicable to this PR.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n\r\n\r\n### For
maintainers\r\n\r\n- [ ] This was checked for breaking API changes and
was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"f5c78b966b32ed3077b7795308f8902d1120013f"}},"sourceBranch":"main","suggestedTargetBranches":["8.12"],"targetPullRequestStates":[{"branch":"8.12","label":"v8.12.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Chenhui Wang <[email protected]>

x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createCollectorFetchContextMock } from '@kbn/usage-collection-plugin/server/mocks';
+
+import { registerTelemetryUsageCollector } from './telemetry';
+
+describe('Connectors Telemetry Usage Collector', () => {
+  const makeUsageCollectorStub = jest.fn();
+  const registerStub = jest.fn();
+  const usageCollectionMock = {
+    makeUsageCollector: makeUsageCollectorStub,
+    registerCollector: registerStub,
+  } as any;
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('registerTelemetryUsageCollector', () => {
+    it('should make and register the usage collector', () => {
+      registerTelemetryUsageCollector(usageCollectionMock);
+
+      expect(registerStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('connectors');
+      expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true);
+    });
+  });
+
+  describe('fetchTelemetryMetrics', () => {
+    it('should return telemetry data', async () => {
+      const fetchContextMock = createCollectorFetchContextMock();
+      fetchContextMock.esClient.count = jest.fn().mockImplementation((query: any) =>
+        Promise.resolve({
+          count: query.query.bool.filter[0].term.is_native ? 5 : 2,
+        })
+      );
+      registerTelemetryUsageCollector(usageCollectionMock);
+      const telemetryMetrics = await makeUsageCollectorStub.mock.calls[0][0].fetch(
+        fetchContextMock
+      );
+
+      expect(telemetryMetrics).toEqual({
+        native: {
+          total: 5,
+        },
+        clients: {
+          total: 2,
+        },
+      });
+    });
+  });
+});

x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+import { CONNECTORS_INDEX } from '@kbn/search-connectors';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+
+interface Telemetry {
+  native: {
+    total: number;
+  };
+  clients: {
+    total: number;
+  };
+}
+
+/**
+ * Register the telemetry collector
+ */
+
+export const registerTelemetryUsageCollector = (usageCollection: UsageCollectionSetup) => {
+  const telemetryUsageCollector = usageCollection.makeUsageCollector<Telemetry>({
+    type: 'connectors',
+    isReady: () => true,
+    schema: {
+      native: {
+        total: { type: 'long' },
+      },
+      clients: {
+        total: { type: 'long' },
+      },
+    },
+    async fetch({ esClient }) {
+      return await fetchTelemetryMetrics(esClient);
+    },
+  });
+  usageCollection.registerCollector(telemetryUsageCollector);
+};
+
+/**
+ * Fetch the aggregated telemetry metrics
+ */
+
+export const fetchTelemetryMetrics = async (client: ElasticsearchClient): Promise<Telemetry> => {
+  const [nativeCountResponse, clientsCountResponse] = await Promise.all([
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: true,
+              },
+            },
+          ],
+          must_not: [
+            {
+              term: {
+                service_type: {
+                  value: 'elastic-crawler',
+                },
+              },
+            },
+          ],
+        },
+      },
+    }),
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: false,
+              },
+            },
+          ],
+        },
+      },
+    }),
+  ]);
+
+  return {
+    native: {
+      total: nativeCountResponse.count,
+    },
+    clients: {
+      total: clientsCountResponse.count,
+    },
+  } as Telemetry;
+};

x-pack/plugins/enterprise_search/server/plugin.ts

@@ -50,6 +50,7 @@ import {
 } from '../common/guided_onboarding/search_guide_config';
 
 import { registerTelemetryUsageCollector as registerASTelemetryUsageCollector } from './collectors/app_search/telemetry';
+import { registerTelemetryUsageCollector as registerCNTelemetryUsageCollector } from './collectors/connectors/telemetry';
 import { registerTelemetryUsageCollector as registerESTelemetryUsageCollector } from './collectors/enterprise_search/telemetry';
 import { registerTelemetryUsageCollector as registerWSTelemetryUsageCollector } from './collectors/workplace_search/telemetry';
 import { registerEnterpriseSearchIntegrations } from './integrations';
@@ -262,6 +263,7 @@ export class EnterpriseSearchPlugin implements Plugin {
 
       if (usageCollection) {
         registerESTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger);
+        registerCNTelemetryUsageCollector(usageCollection);
         if (config.canDeployEntSearch) {
           registerASTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger);
           registerWSTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger);

x-pack/plugins/serverless_search/kibana.jsonc

@@ -28,6 +28,7 @@
     ],
     "optionalPlugins": [
       "indexManagement",
+      "usageCollection",
     ],
     "requiredBundles": [
       "kibanaReact"

x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { registerTelemetryUsageCollector } from './telemetry';
+import { createCollectorFetchContextMock } from '@kbn/usage-collection-plugin/server/mocks';
+
+describe('Connectors Serverless Telemetry Usage Collector', () => {
+  const makeUsageCollectorStub = jest.fn();
+  const registerStub = jest.fn();
+  const usageCollectionMock = {
+    makeUsageCollector: makeUsageCollectorStub,
+    registerCollector: registerStub,
+  } as any;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('registerTelemetryUsageCollector', () => {
+    it('should make and register the usage collector', () => {
+      registerTelemetryUsageCollector(usageCollectionMock);
+
+      expect(registerStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1);
+      expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('connectors_serverless');
+      expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true);
+    });
+  });
+
+  describe('fetchTelemetryMetrics', () => {
+    it('should return telemetry data', async () => {
+      const fetchContextMock = createCollectorFetchContextMock();
+      fetchContextMock.esClient.count = jest.fn().mockImplementation((query: any) =>
+        Promise.resolve({
+          count: query.query.bool.filter[0].term.is_native ? 5 : 2,
+        })
+      );
+      registerTelemetryUsageCollector(usageCollectionMock);
+      const telemetryMetrics = await makeUsageCollectorStub.mock.calls[0][0].fetch(
+        fetchContextMock
+      );
+
+      expect(telemetryMetrics).toEqual({
+        native: {
+          total: 5,
+        },
+        clients: {
+          total: 2,
+        },
+      });
+    });
+  });
+});

x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+
+import { CONNECTORS_INDEX } from '@kbn/search-connectors';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+
+interface Telemetry {
+  native: {
+    total: number;
+  };
+  clients: {
+    total: number;
+  };
+}
+
+/**
+ * Register the telemetry collector
+ */
+
+export const registerTelemetryUsageCollector = (usageCollection: UsageCollectionSetup) => {
+  const telemetryUsageCollector = usageCollection.makeUsageCollector<Telemetry>({
+    type: 'connectors_serverless',
+    isReady: () => true,
+    schema: {
+      native: {
+        total: { type: 'long' },
+      },
+      clients: {
+        total: { type: 'long' },
+      },
+    },
+    async fetch({ esClient }) {
+      return await fetchTelemetryMetrics(esClient);
+    },
+  });
+  usageCollection.registerCollector(telemetryUsageCollector);
+};
+
+/**
+ * Fetch the aggregated telemetry metrics
+ */
+
+export const fetchTelemetryMetrics = async (client: ElasticsearchClient): Promise<Telemetry> => {
+  const [nativeCountResponse, clientsCountResponse] = await Promise.all([
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: true,
+              },
+            },
+          ],
+          must_not: [
+            {
+              term: {
+                service_type: {
+                  value: 'elastic-crawler',
+                },
+              },
+            },
+          ],
+        },
+      },
+    }),
+    client.count({
+      index: CONNECTORS_INDEX,
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                is_native: false,
+              },
+            },
+          ],
+        },
+      },
+    }),
+  ]);
+
+  return {
+    native: {
+      total: nativeCountResponse.count,
+    },
+    clients: {
+      total: clientsCountResponse.count,
+    },
+  } as Telemetry;
+};

x-pack/plugins/serverless_search/server/plugin.ts

@@ -27,6 +27,7 @@ import type {
   StartDependencies,
 } from './types';
 import { registerConnectorsRoutes } from './routes/connectors_routes';
+import { registerTelemetryUsageCollector } from './collectors/connectors/telemetry';
 
 export interface RouteDependencies {
   http: CoreSetup<StartDependencies>['http'];
@@ -77,7 +78,7 @@ export class ServerlessSearchPlugin
 
   public setup(
     { getStartServices, http }: CoreSetup<StartDependencies>,
-    pluginsSetup: SetupDependencies
+    { serverless, usageCollection }: SetupDependencies
   ) {
     const router = http.createRouter();
     getStartServices().then(([, { security }]) => {
@@ -94,7 +95,13 @@ export class ServerlessSearchPlugin
       registerIndicesRoutes(dependencies);
     });
 
-    pluginsSetup.serverless.setupProjectSettings(SEARCH_PROJECT_SETTINGS);
+    if (usageCollection) {
+      getStartServices().then(() => {
+        registerTelemetryUsageCollector(usageCollection);
+      });
+    }
+
+    serverless.setupProjectSettings(SEARCH_PROJECT_SETTINGS);
     return {};
   }
 

x-pack/plugins/serverless_search/server/types.ts

@@ -8,6 +8,7 @@
 import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
 import type { SecurityPluginStart } from '@kbn/security-plugin/server';
 import type { ServerlessPluginSetup } from '@kbn/serverless/server';
+import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface ServerlessSearchPluginSetup {}
@@ -20,4 +21,5 @@ export interface StartDependencies {
 }
 export interface SetupDependencies {
   serverless: ServerlessPluginSetup;
+  usageCollection?: UsageCollectionSetup;
 }

x-pack/plugins/serverless_search/tsconfig.json

@@ -38,5 +38,6 @@
     "@kbn/data-views-plugin",
     "@kbn/kibana-utils-plugin",
     "@kbn/index-management-plugin",
+    "@kbn/usage-collection-plugin",
   ]
 }