From f40e47cc108b0112cf916d917937aa685211f397 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Dec 2023 03:18:10 -0500 Subject: [PATCH] [8.12] Create usage collectors for connectors (#172280) (#172774) # Backport This will backport the following commits from `main` to `8.12`: - [Create usage collectors for connectors (#172280)](https://github.com/elastic/kibana/pull/172280) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Chenhui Wang <54903978+wangch079@users.noreply.github.com> --- .../collectors/connectors/telemetry.test.ts | 57 +++++++++++ .../server/collectors/connectors/telemetry.ts | 98 +++++++++++++++++++ .../enterprise_search/server/plugin.ts | 2 + x-pack/plugins/serverless_search/kibana.jsonc | 1 + .../collectors/connectors/telemetry.test.ts | 57 +++++++++++ .../server/collectors/connectors/telemetry.ts | 98 +++++++++++++++++++ .../serverless_search/server/plugin.ts | 11 ++- .../plugins/serverless_search/server/types.ts | 2 + .../plugins/serverless_search/tsconfig.json | 1 + .../schema/xpack_plugins.json | 36 +++++++ 10 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts create mode 100644 x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts create mode 100644 x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts create mode 100644 x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts diff --git a/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts b/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts new file mode 100644 index 0000000000000..64078cfbb20e9 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createCollectorFetchContextMock } from '@kbn/usage-collection-plugin/server/mocks'; + +import { registerTelemetryUsageCollector } from './telemetry'; + +describe('Connectors Telemetry Usage Collector', () => { + const makeUsageCollectorStub = jest.fn(); + const registerStub = jest.fn(); + const usageCollectionMock = { + makeUsageCollector: makeUsageCollectorStub, + registerCollector: registerStub, + } as any; + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('registerTelemetryUsageCollector', () => { + it('should make and register the usage collector', () => { + registerTelemetryUsageCollector(usageCollectionMock); + + expect(registerStub).toHaveBeenCalledTimes(1); + expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1); + expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('connectors'); + expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true); + }); + }); + + describe('fetchTelemetryMetrics', () => { + it('should return telemetry data', async () => { + const fetchContextMock = createCollectorFetchContextMock(); + fetchContextMock.esClient.count = jest.fn().mockImplementation((query: any) => + Promise.resolve({ + count: query.query.bool.filter[0].term.is_native ? 5 : 2, + }) + ); + registerTelemetryUsageCollector(usageCollectionMock); + const telemetryMetrics = await makeUsageCollectorStub.mock.calls[0][0].fetch( + fetchContextMock + ); + + expect(telemetryMetrics).toEqual({ + native: { + total: 5, + }, + clients: { + total: 2, + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts b/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts new file mode 100644 index 0000000000000..d5a61dff0a692 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/collectors/connectors/telemetry.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; + +import { CONNECTORS_INDEX } from '@kbn/search-connectors'; +import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; + +interface Telemetry { + native: { + total: number; + }; + clients: { + total: number; + }; +} + +/** + * Register the telemetry collector + */ + +export const registerTelemetryUsageCollector = (usageCollection: UsageCollectionSetup) => { + const telemetryUsageCollector = usageCollection.makeUsageCollector({ + type: 'connectors', + isReady: () => true, + schema: { + native: { + total: { type: 'long' }, + }, + clients: { + total: { type: 'long' }, + }, + }, + async fetch({ esClient }) { + return await fetchTelemetryMetrics(esClient); + }, + }); + usageCollection.registerCollector(telemetryUsageCollector); +}; + +/** + * Fetch the aggregated telemetry metrics + */ + +export const fetchTelemetryMetrics = async (client: ElasticsearchClient): Promise => { + const [nativeCountResponse, clientsCountResponse] = await Promise.all([ + client.count({ + index: CONNECTORS_INDEX, + query: { + bool: { + filter: [ + { + term: { + is_native: true, + }, + }, + ], + must_not: [ + { + term: { + service_type: { + value: 'elastic-crawler', + }, + }, + }, + ], + }, + }, + }), + client.count({ + index: CONNECTORS_INDEX, + query: { + bool: { + filter: [ + { + term: { + is_native: false, + }, + }, + ], + }, + }, + }), + ]); + + return { + native: { + total: nativeCountResponse.count, + }, + clients: { + total: clientsCountResponse.count, + }, + } as Telemetry; +}; diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts index 6231673a8a153..1132de4fbcccf 100644 --- a/x-pack/plugins/enterprise_search/server/plugin.ts +++ b/x-pack/plugins/enterprise_search/server/plugin.ts @@ -50,6 +50,7 @@ import { } from '../common/guided_onboarding/search_guide_config'; import { registerTelemetryUsageCollector as registerASTelemetryUsageCollector } from './collectors/app_search/telemetry'; +import { registerTelemetryUsageCollector as registerCNTelemetryUsageCollector } from './collectors/connectors/telemetry'; import { registerTelemetryUsageCollector as registerESTelemetryUsageCollector } from './collectors/enterprise_search/telemetry'; import { registerTelemetryUsageCollector as registerWSTelemetryUsageCollector } from './collectors/workplace_search/telemetry'; import { registerEnterpriseSearchIntegrations } from './integrations'; @@ -262,6 +263,7 @@ export class EnterpriseSearchPlugin implements Plugin { if (usageCollection) { registerESTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger); + registerCNTelemetryUsageCollector(usageCollection); if (config.canDeployEntSearch) { registerASTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger); registerWSTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger); diff --git a/x-pack/plugins/serverless_search/kibana.jsonc b/x-pack/plugins/serverless_search/kibana.jsonc index a9000e7542475..cc81622911a9a 100644 --- a/x-pack/plugins/serverless_search/kibana.jsonc +++ b/x-pack/plugins/serverless_search/kibana.jsonc @@ -28,6 +28,7 @@ ], "optionalPlugins": [ "indexManagement", + "usageCollection", ], "requiredBundles": [ "kibanaReact" diff --git a/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts b/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts new file mode 100644 index 0000000000000..7c3d693f428de --- /dev/null +++ b/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { registerTelemetryUsageCollector } from './telemetry'; +import { createCollectorFetchContextMock } from '@kbn/usage-collection-plugin/server/mocks'; + +describe('Connectors Serverless Telemetry Usage Collector', () => { + const makeUsageCollectorStub = jest.fn(); + const registerStub = jest.fn(); + const usageCollectionMock = { + makeUsageCollector: makeUsageCollectorStub, + registerCollector: registerStub, + } as any; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('registerTelemetryUsageCollector', () => { + it('should make and register the usage collector', () => { + registerTelemetryUsageCollector(usageCollectionMock); + + expect(registerStub).toHaveBeenCalledTimes(1); + expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1); + expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('connectors_serverless'); + expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true); + }); + }); + + describe('fetchTelemetryMetrics', () => { + it('should return telemetry data', async () => { + const fetchContextMock = createCollectorFetchContextMock(); + fetchContextMock.esClient.count = jest.fn().mockImplementation((query: any) => + Promise.resolve({ + count: query.query.bool.filter[0].term.is_native ? 5 : 2, + }) + ); + registerTelemetryUsageCollector(usageCollectionMock); + const telemetryMetrics = await makeUsageCollectorStub.mock.calls[0][0].fetch( + fetchContextMock + ); + + expect(telemetryMetrics).toEqual({ + native: { + total: 5, + }, + clients: { + total: 2, + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts b/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts new file mode 100644 index 0000000000000..edf95fc197528 --- /dev/null +++ b/x-pack/plugins/serverless_search/server/collectors/connectors/telemetry.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; + +import { CONNECTORS_INDEX } from '@kbn/search-connectors'; +import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; + +interface Telemetry { + native: { + total: number; + }; + clients: { + total: number; + }; +} + +/** + * Register the telemetry collector + */ + +export const registerTelemetryUsageCollector = (usageCollection: UsageCollectionSetup) => { + const telemetryUsageCollector = usageCollection.makeUsageCollector({ + type: 'connectors_serverless', + isReady: () => true, + schema: { + native: { + total: { type: 'long' }, + }, + clients: { + total: { type: 'long' }, + }, + }, + async fetch({ esClient }) { + return await fetchTelemetryMetrics(esClient); + }, + }); + usageCollection.registerCollector(telemetryUsageCollector); +}; + +/** + * Fetch the aggregated telemetry metrics + */ + +export const fetchTelemetryMetrics = async (client: ElasticsearchClient): Promise => { + const [nativeCountResponse, clientsCountResponse] = await Promise.all([ + client.count({ + index: CONNECTORS_INDEX, + query: { + bool: { + filter: [ + { + term: { + is_native: true, + }, + }, + ], + must_not: [ + { + term: { + service_type: { + value: 'elastic-crawler', + }, + }, + }, + ], + }, + }, + }), + client.count({ + index: CONNECTORS_INDEX, + query: { + bool: { + filter: [ + { + term: { + is_native: false, + }, + }, + ], + }, + }, + }), + ]); + + return { + native: { + total: nativeCountResponse.count, + }, + clients: { + total: clientsCountResponse.count, + }, + } as Telemetry; +}; diff --git a/x-pack/plugins/serverless_search/server/plugin.ts b/x-pack/plugins/serverless_search/server/plugin.ts index 98f60c803972b..6e6a90735ee23 100644 --- a/x-pack/plugins/serverless_search/server/plugin.ts +++ b/x-pack/plugins/serverless_search/server/plugin.ts @@ -27,6 +27,7 @@ import type { StartDependencies, } from './types'; import { registerConnectorsRoutes } from './routes/connectors_routes'; +import { registerTelemetryUsageCollector } from './collectors/connectors/telemetry'; export interface RouteDependencies { http: CoreSetup['http']; @@ -77,7 +78,7 @@ export class ServerlessSearchPlugin public setup( { getStartServices, http }: CoreSetup, - pluginsSetup: SetupDependencies + { serverless, usageCollection }: SetupDependencies ) { const router = http.createRouter(); getStartServices().then(([, { security }]) => { @@ -94,7 +95,13 @@ export class ServerlessSearchPlugin registerIndicesRoutes(dependencies); }); - pluginsSetup.serverless.setupProjectSettings(SEARCH_PROJECT_SETTINGS); + if (usageCollection) { + getStartServices().then(() => { + registerTelemetryUsageCollector(usageCollection); + }); + } + + serverless.setupProjectSettings(SEARCH_PROJECT_SETTINGS); return {}; } diff --git a/x-pack/plugins/serverless_search/server/types.ts b/x-pack/plugins/serverless_search/server/types.ts index 3cbd26ad2dd5f..a78539c5b89e2 100644 --- a/x-pack/plugins/serverless_search/server/types.ts +++ b/x-pack/plugins/serverless_search/server/types.ts @@ -8,6 +8,7 @@ import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; import type { SecurityPluginStart } from '@kbn/security-plugin/server'; import type { ServerlessPluginSetup } from '@kbn/serverless/server'; +import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ServerlessSearchPluginSetup {} @@ -20,4 +21,5 @@ export interface StartDependencies { } export interface SetupDependencies { serverless: ServerlessPluginSetup; + usageCollection?: UsageCollectionSetup; } diff --git a/x-pack/plugins/serverless_search/tsconfig.json b/x-pack/plugins/serverless_search/tsconfig.json index f58828f69569b..3f79fa5c9277a 100644 --- a/x-pack/plugins/serverless_search/tsconfig.json +++ b/x-pack/plugins/serverless_search/tsconfig.json @@ -38,5 +38,6 @@ "@kbn/data-views-plugin", "@kbn/kibana-utils-plugin", "@kbn/index-management-plugin", + "@kbn/usage-collection-plugin", ] } diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 743e19137ff19..777d78acb192f 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -7713,6 +7713,42 @@ } } }, + "connectors": { + "properties": { + "native": { + "properties": { + "total": { + "type": "long" + } + } + }, + "clients": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "connectors_serverless": { + "properties": { + "native": { + "properties": { + "total": { + "type": "long" + } + } + }, + "clients": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, "discoverEnhanced": { "properties": { "exploreDataInChartActionEnabled": {