[8.x] Push the total request-body-bytes to usage-api (#194429) (#198614)

# Backport

This will backport the following commits from `main` to `8.x`:
- [Push the total request-body-bytes to usage-api
(#194429)](#194429)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Ersin
Erdal","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-31T19:30:16Z","message":"Push
the total request-body-bytes to usage-api (#194429)\n\nResolves:
elastic/response-ops-team#209 \r\n\r\nThis PR
is a follow-on of
https://github.com/elastic/kibana/pull/186804.\r\n\r\nCreates a new task
that runs every 1 hour to push the total\r\nconnector-request-body-bytes
that have been saved in the event log
to\r\nusage-api.","sha":"216f8996214cb93c2a44ff85fa844f8e428017b7","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:ResponseOps","v9.0.0","backport:prev-minor","ci:project-deploy-observability"],"title":"Push
the total request-body-bytes to
usage-api","number":194429,"url":"https://github.com/elastic/kibana/pull/194429","mergeCommit":{"message":"Push
the total request-body-bytes to usage-api (#194429)\n\nResolves:
elastic/response-ops-team#209 \r\n\r\nThis PR
is a follow-on of
https://github.com/elastic/kibana/pull/186804.\r\n\r\nCreates a new task
that runs every 1 hour to push the total\r\nconnector-request-body-bytes
that have been saved in the event log
to\r\nusage-api.","sha":"216f8996214cb93c2a44ff85fa844f8e428017b7"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/194429","number":194429,"mergeCommit":{"message":"Push
the total request-body-bytes to usage-api (#194429)\n\nResolves:
elastic/response-ops-team#209 \r\n\r\nThis PR
is a follow-on of
https://github.com/elastic/kibana/pull/186804.\r\n\r\nCreates a new task
that runs every 1 hour to push the total\r\nconnector-request-body-bytes
that have been saved in the event log
to\r\nusage-api.","sha":"216f8996214cb93c2a44ff85fa844f8e428017b7"}}]}]
BACKPORT-->

Co-authored-by: Ersin Erdal <[email protected]>

x-pack/plugins/actions/kibana.jsonc

@@ -22,7 +22,8 @@
       "spaces",
       "security",
       "monitoringCollection",
-      "serverless"
+      "serverless",
+      "cloud"
     ],
     "extraPublicDirs": [
       "common"

x-pack/plugins/actions/server/actions_client/actions_client.test.ts

@@ -57,6 +57,7 @@ import { OAuthParams } from '../routes/get_oauth_access_token';
 import { eventLogClientMock } from '@kbn/event-log-plugin/server/event_log_client.mock';
 import { GetGlobalExecutionKPIParams, GetGlobalExecutionLogParams } from '../../common';
 import { estypes } from '@elastic/elasticsearch';
+import { DEFAULT_USAGE_API_URL } from '../config';
 
 jest.mock('@kbn/core-saved-objects-utils-server', () => {
   const actual = jest.requireActual('@kbn/core-saved-objects-utils-server');
@@ -613,6 +614,9 @@ describe('create()', () => {
       microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
       microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
       microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+      usage: {
+        url: DEFAULT_USAGE_API_URL,
+      },
     });
 
     const localActionTypeRegistryParams = {

x-pack/plugins/actions/server/actions_config.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { ByteSizeValue } from '@kbn/config-schema';
-import { ActionsConfig } from './config';
+import { ActionsConfig, DEFAULT_USAGE_API_URL } from './config';
 import {
   DEFAULT_MICROSOFT_EXCHANGE_URL,
   DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
@@ -42,6 +42,9 @@ const defaultActionsConfig: ActionsConfig = {
   microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
   microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
   microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+  usage: {
+    url: DEFAULT_USAGE_API_URL,
+  },
 };
 
 describe('ensureUriAllowed', () => {

x-pack/plugins/actions/server/config.test.ts

@@ -38,6 +38,9 @@ describe('config validation', () => {
         "proxyRejectUnauthorizedCertificates": true,
         "rejectUnauthorized": true,
         "responseTimeout": "PT1M",
+        "usage": Object {
+          "url": "https://usage-api.usage-api/api/v1/usage",
+        },
       }
     `);
   });
@@ -85,6 +88,9 @@ describe('config validation', () => {
         "proxyRejectUnauthorizedCertificates": false,
         "rejectUnauthorized": false,
         "responseTimeout": "PT1M",
+        "usage": Object {
+          "url": "https://usage-api.usage-api/api/v1/usage",
+        },
       }
     `);
   });
@@ -225,6 +231,9 @@ describe('config validation', () => {
           "proxyVerificationMode": "none",
           "verificationMode": "none",
         },
+        "usage": Object {
+          "url": "https://usage-api.usage-api/api/v1/usage",
+        },
       }
     `);
   });

x-pack/plugins/actions/server/config.ts

@@ -72,6 +72,8 @@ const connectorTypeSchema = schema.object({
   maxAttempts: schema.maybe(schema.number({ min: MIN_MAX_ATTEMPTS, max: MAX_MAX_ATTEMPTS })),
 });
 
+export const DEFAULT_USAGE_API_URL = 'https://usage-api.usage-api/api/v1/usage';
+
 // We leverage enabledActionTypes list by allowing the other plugins to overwrite it by using "setEnabledConnectorTypes" in the plugin setup.
 // The list can be overwritten only if it's not already been set in the config.
 const enabledConnectorTypesSchema = schema.arrayOf(
@@ -145,15 +147,14 @@ export const configSchema = schema.object({
       max: schema.maybe(schema.number({ min: MIN_QUEUED_MAX, defaultValue: DEFAULT_QUEUED_MAX })),
     })
   ),
-  usage: schema.maybe(
-    schema.object({
-      ca: schema.maybe(
-        schema.object({
-          path: schema.string(),
-        })
-      ),
-    })
-  ),
+  usage: schema.object({
+    url: schema.string({ defaultValue: DEFAULT_USAGE_API_URL }),
+    ca: schema.maybe(
+      schema.object({
+        path: schema.string(),
+      })
+    ),
+  }),
 });
 
 export type ActionsConfig = TypeOf<typeof configSchema>;

x-pack/plugins/actions/server/integration_tests/axios_utils_connection.test.ts

@@ -20,7 +20,7 @@ import { ByteSizeValue } from '@kbn/config-schema';
 import { Logger } from '@kbn/core/server';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 import { createReadySignal } from '@kbn/event-log-plugin/server/lib/ready_signal';
-import { ActionsConfig } from '../config';
+import { ActionsConfig, DEFAULT_USAGE_API_URL } from '../config';
 import { ActionsConfigurationUtilities, getActionsConfigurationUtilities } from '../actions_config';
 import { resolveCustomHosts } from '../lib/custom_host_settings';
 import {
@@ -691,6 +691,9 @@ const BaseActionsConfig: ActionsConfig = {
   microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
   microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
   microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+  usage: {
+    url: DEFAULT_USAGE_API_URL,
+  },
 };
 
 function getACUfromConfig(config: Partial<ActionsConfig> = {}): ActionsConfigurationUtilities {

x-pack/plugins/actions/server/integration_tests/axios_utils_proxy.test.ts

@@ -20,7 +20,7 @@ import { ByteSizeValue } from '@kbn/config-schema';
 import { Logger } from '@kbn/core/server';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 import { createReadySignal } from '@kbn/event-log-plugin/server/lib/ready_signal';
-import { ActionsConfig } from '../config';
+import { ActionsConfig, DEFAULT_USAGE_API_URL } from '../config';
 import { ActionsConfigurationUtilities, getActionsConfigurationUtilities } from '../actions_config';
 import { resolveCustomHosts } from '../lib/custom_host_settings';
 import {
@@ -597,6 +597,9 @@ const BaseActionsConfig: ActionsConfig = {
   microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
   microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
   microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+  usage: {
+    url: DEFAULT_USAGE_API_URL,
+  },
 };
 
 function getACUfromConfig(config: Partial<ActionsConfig> = {}): ActionsConfigurationUtilities {

x-pack/plugins/actions/server/lib/custom_host_settings.test.ts

@@ -10,7 +10,7 @@ import { resolve as pathResolve, join as pathJoin } from 'path';
 import { ByteSizeValue } from '@kbn/config-schema';
 import moment from 'moment';
 
-import { ActionsConfig } from '../config';
+import { ActionsConfig, DEFAULT_USAGE_API_URL } from '../config';
 import { Logger } from '@kbn/core/server';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 
@@ -82,6 +82,9 @@ describe('custom_host_settings', () => {
       microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
       microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
       microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+      usage: {
+        url: DEFAULT_USAGE_API_URL,
+      },
     };
 
     test('ensure it copies over the config parts that it does not touch', () => {

x-pack/plugins/actions/server/plugin.test.ts

@@ -30,6 +30,7 @@ import {
   DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
   DEFAULT_MICROSOFT_GRAPH_API_URL,
 } from '../common';
+import { cloudMock } from '@kbn/cloud-plugin/server/mocks';
 
 const executor: ExecutorType<{}, {}, {}, void> = async (options) => {
   return { status: 'ok', actionId: options.actionId };
@@ -59,6 +60,9 @@ function getConfig(overrides = {}) {
     microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
     microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
     microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+    usage: {
+      url: 'ca.path',
+    },
     ...overrides,
   };
 }
@@ -84,6 +88,9 @@ describe('Actions Plugin', () => {
         microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
         microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
         microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+        usage: {
+          url: 'ca.path',
+        },
       });
       plugin = new ActionsPlugin(context);
       coreSetup = coreMock.createSetup();
@@ -95,6 +102,7 @@ describe('Actions Plugin', () => {
         eventLog: eventLogMock.createSetup(),
         usageCollection: usageCollectionPluginMock.createSetupContract(),
         features: featuresPluginMock.createSetup(),
+        cloud: cloudMock.createSetup(),
       };
       coreSetup.getStartServices.mockResolvedValue([
         coreMock.createStart(),
@@ -347,6 +355,7 @@ describe('Actions Plugin', () => {
           eventLog: eventLogMock.createSetup(),
           usageCollection: usageCollectionPluginMock.createSetupContract(),
           features: featuresPluginMock.createSetup(),
+          cloud: cloudMock.createSetup(),
         };
       }
 
@@ -374,6 +383,7 @@ describe('Actions Plugin', () => {
           usageCollection: usageCollectionPluginMock.createSetupContract(),
           features: featuresPluginMock.createSetup(),
           serverless: serverlessPluginMock.createSetupContract(),
+          cloud: cloudMock.createSetup(),
         };
       }
 
@@ -585,6 +595,9 @@ describe('Actions Plugin', () => {
         microsoftGraphApiUrl: DEFAULT_MICROSOFT_GRAPH_API_URL,
         microsoftGraphApiScope: DEFAULT_MICROSOFT_GRAPH_API_SCOPE,
         microsoftExchangeUrl: DEFAULT_MICROSOFT_EXCHANGE_URL,
+        usage: {
+          url: 'ca.path',
+        },
       });
       plugin = new ActionsPlugin(context);
       coreSetup = coreMock.createSetup();
@@ -596,6 +609,7 @@ describe('Actions Plugin', () => {
         eventLog: eventLogMock.createSetup(),
         usageCollection: usageCollectionPluginMock.createSetupContract(),
         features: featuresPluginMock.createSetup(),
+        cloud: cloudMock.createSetup(),
       };
       pluginsStart = {
         licensing: licensingMock.createStart(),
@@ -680,6 +694,7 @@ describe('Actions Plugin', () => {
           eventLog: eventLogMock.createSetup(),
           usageCollection: usageCollectionPluginMock.createSetupContract(),
           features: featuresPluginMock.createSetup(),
+          cloud: cloudMock.createSetup(),
         };
         pluginsStart = {
           licensing: licensingMock.createStart(),

x-pack/plugins/actions/server/plugin.ts

@@ -42,6 +42,7 @@ import {
 import { MonitoringCollectionSetup } from '@kbn/monitoring-collection-plugin/server';
 
 import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/server';
+import type { CloudSetup } from '@kbn/cloud-plugin/server';
 import { ActionsConfig, AllowedHosts, EnabledConnectorTypes, getValidatedConfig } from './config';
 import { resolveCustomHosts } from './lib/custom_host_settings';
 import { events } from './lib/event_based_telemetry';
@@ -112,6 +113,7 @@ import type { IUnsecuredActionsClient } from './unsecured_actions_client/unsecur
 import { UnsecuredActionsClient } from './unsecured_actions_client/unsecured_actions_client';
 import { createBulkUnsecuredExecutionEnqueuerFunction } from './create_unsecured_execute_function';
 import { createSystemConnectors } from './create_system_actions';
+import { ConnectorUsageReportingTask } from './usage/connector_usage_reporting_task';
 
 export interface PluginSetupContract {
   registerType<
@@ -184,6 +186,7 @@ export interface ActionsPluginsSetup {
   spaces?: SpacesPluginSetup;
   monitoringCollection?: MonitoringCollectionSetup;
   serverless?: ServerlessPluginSetup;
+  cloud: CloudSetup;
 }
 
 export interface ActionsPluginsStart {
@@ -218,6 +221,7 @@ export class ActionsPlugin implements Plugin<PluginSetupContract, PluginStartCon
   private readonly telemetryLogger: Logger;
   private inMemoryConnectors: InMemoryConnector[];
   private inMemoryMetrics: InMemoryMetrics;
+  private connectorUsageReportingTask: ConnectorUsageReportingTask | undefined;
 
   constructor(initContext: PluginInitializerContext) {
     this.logger = initContext.logger.get();
@@ -327,6 +331,15 @@ export class ActionsPlugin implements Plugin<PluginSetupContract, PluginStartCon
         this.getInMemoryConnectors,
         eventLogIndex
       );
+
+      this.connectorUsageReportingTask = new ConnectorUsageReportingTask({
+        logger: this.logger,
+        eventLogIndex,
+        core,
+        taskManager: plugins.taskManager,
+        projectId: plugins.cloud.serverless.projectId,
+        config: this.actionsConfig.usage,
+      });
     }
 
     // Usage counter for telemetry
@@ -603,6 +616,8 @@ export class ActionsPlugin implements Plugin<PluginSetupContract, PluginStartCon
 
     this.validateEnabledConnectorTypes(plugins);
 
+    this.connectorUsageReportingTask?.start(plugins.taskManager).catch(() => {});
+
     return {
       isActionTypeEnabled: (id, options = { notifyUsage: false }) => {
         return this.actionTypeRegistry!.isActionTypeEnabled(id, options);