From 4b9256f202fba7e433bd477f8aaa4d3d70d96413 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:09:53 +1100 Subject: [PATCH] [8.16] [Data Usage] use auto ops service (#196312) (#197132) # Backport This will backport the following commits from `main` to `8.16`: - [[Data Usage] use auto ops service (#196312)](https://github.com/elastic/kibana/pull/196312) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\nCo-authored-by: Sandra G "}}]}] BACKPORT--> Co-authored-by: Sandra G --- .../public/hooks/use_get_data_streams.ts | 12 ++ .../public/hooks/use_get_usage_metrics.ts | 12 ++ x-pack/plugins/data_usage/server/plugin.ts | 16 +- .../data_usage/server/routes/index.tsx | 9 +- .../server/routes/internal/data_streams.ts | 38 ++-- .../routes/internal/data_streams_handler.ts | 7 +- .../server/routes/internal/usage_metrics.ts | 41 +++-- .../routes/internal/usage_metrics_handler.ts | 165 +----------------- .../server/{ => services}/app_context.ts | 10 +- .../data_usage/server/services/autoops_api.ts | 24 +-- .../data_usage/server/services/index.ts | 43 +++++ 11 files changed, 140 insertions(+), 237 deletions(-) rename x-pack/plugins/data_usage/server/{ => services}/app_context.ts (89%) create mode 100644 x-pack/plugins/data_usage/server/services/index.ts diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_data_streams.ts b/x-pack/plugins/data_usage/public/hooks/use_get_data_streams.ts index 46a448ac82b31..35f53c49e2c28 100644 --- a/x-pack/plugins/data_usage/public/hooks/use_get_data_streams.ts +++ b/x-pack/plugins/data_usage/public/hooks/use_get_data_streams.ts @@ -6,6 +6,7 @@ */ import type { UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; import { useQuery } from '@tanstack/react-query'; import type { IHttpFetchError } from '@kbn/core-http-browser'; import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../common'; @@ -32,6 +33,9 @@ export const useGetDataUsageDataStreams = ({ options?: UseQueryOptions; }): UseQueryResult => { const http = useKibanaContextForPlugin().services.http; + const { + services: { notifications }, + } = useKibanaContextForPlugin(); return useQuery({ queryKey: ['get-data-usage-data-streams'], @@ -83,5 +87,13 @@ export const useGetDataUsageDataStreams = ({ : PAGING_PARAMS.default ); }, + onError: (error: IHttpFetchError) => { + notifications.toasts.addDanger({ + title: i18n.translate('xpack.dataUsage.getDataStreams.addFailure.toast.title', { + defaultMessage: 'Error getting data streams', + }), + text: error.message, + }); + }, }); }; diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts index 4e89a7a3f5f0e..bbd0f5d8aa02f 100644 --- a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -6,6 +6,7 @@ */ import type { UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; import { useQuery } from '@tanstack/react-query'; import type { IHttpFetchError } from '@kbn/core-http-browser'; import { UsageMetricsRequestBody, UsageMetricsResponseSchemaBody } from '../../common/rest_types'; @@ -22,6 +23,9 @@ export const useGetDataUsageMetrics = ( options: UseQueryOptions> = {} ): UseQueryResult> => { const http = useKibanaContextForPlugin().services.http; + const { + services: { notifications }, + } = useKibanaContextForPlugin(); return useQuery>({ queryKey: ['get-data-usage-metrics', body], @@ -39,5 +43,13 @@ export const useGetDataUsageMetrics = ( }), }); }, + onError: (error: IHttpFetchError) => { + notifications.toasts.addDanger({ + title: i18n.translate('xpack.dataUsage.getMetrics.addFailure.toast.title', { + defaultMessage: 'Error getting usage metrics', + }), + text: error.message, + }); + }, }); }; diff --git a/x-pack/plugins/data_usage/server/plugin.ts b/x-pack/plugins/data_usage/server/plugin.ts index c282d12d767a1..893b846a0c7e8 100644 --- a/x-pack/plugins/data_usage/server/plugin.ts +++ b/x-pack/plugins/data_usage/server/plugin.ts @@ -18,7 +18,7 @@ import type { } from './types'; import { registerDataUsageRoutes } from './routes'; import { PLUGIN_ID } from '../common'; -import { appContextService } from './app_context'; +import { DataUsageService } from './services'; export class DataUsagePlugin implements @@ -53,6 +53,8 @@ export class DataUsagePlugin } setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup { this.logger.debug('data usage plugin setup'); + const dataUsageService = new DataUsageService(this.dataUsageContext); + pluginsSetup.features.registerElasticsearchFeature({ id: PLUGIN_ID, management: { @@ -66,22 +68,12 @@ export class DataUsagePlugin ], }); const router = coreSetup.http.createRouter(); - registerDataUsageRoutes(router, this.dataUsageContext); + registerDataUsageRoutes(router, dataUsageService); return {}; } start(_coreStart: CoreStart, _pluginsStart: DataUsageStartDependencies): DataUsageServerStart { - appContextService.start({ - logFactory: this.dataUsageContext.logFactory, - configInitialValue: this.dataUsageContext.configInitialValue, - serverConfig: this.dataUsageContext.serverConfig, - config$: this.dataUsageContext.config$, - kibanaVersion: this.dataUsageContext.kibanaVersion, - kibanaBranch: this.dataUsageContext.kibanaBranch, - kibanaInstanceId: this.dataUsageContext.kibanaInstanceId, - cloud: this.dataUsageContext.cloud, - }); return {}; } diff --git a/x-pack/plugins/data_usage/server/routes/index.tsx b/x-pack/plugins/data_usage/server/routes/index.tsx index b6b80c38864f3..ced4f04d034ba 100644 --- a/x-pack/plugins/data_usage/server/routes/index.tsx +++ b/x-pack/plugins/data_usage/server/routes/index.tsx @@ -5,13 +5,14 @@ * 2.0. */ -import { DataUsageContext, DataUsageRouter } from '../types'; +import { DataUsageRouter } from '../types'; import { registerDataStreamsRoute, registerUsageMetricsRoute } from './internal'; +import { DataUsageService } from '../services'; export const registerDataUsageRoutes = ( router: DataUsageRouter, - dataUsageContext: DataUsageContext + dataUsageService: DataUsageService ) => { - registerUsageMetricsRoute(router, dataUsageContext); - registerDataStreamsRoute(router, dataUsageContext); + registerUsageMetricsRoute(router, dataUsageService); + registerDataStreamsRoute(router, dataUsageService); }; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts index 0d71d93b55849..5b972f57984f9 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts @@ -7,31 +7,29 @@ import { DataStreamsResponseSchema } from '../../../common/rest_types'; import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../../common'; -import { DataUsageContext, DataUsageRouter } from '../../types'; - +import { DataUsageRouter } from '../../types'; +import { DataUsageService } from '../../services'; import { getDataStreamsHandler } from './data_streams_handler'; export const registerDataStreamsRoute = ( router: DataUsageRouter, - dataUsageContext: DataUsageContext + dataUsageService: DataUsageService ) => { - if (dataUsageContext.serverConfig.enabled) { - router.versioned - .get({ - access: 'internal', - path: DATA_USAGE_DATA_STREAMS_API_ROUTE, - }) - .addVersion( - { - version: '1', - validate: { - request: {}, - response: { - 200: DataStreamsResponseSchema, - }, + router.versioned + .get({ + access: 'internal', + path: DATA_USAGE_DATA_STREAMS_API_ROUTE, + }) + .addVersion( + { + version: '1', + validate: { + request: {}, + response: { + 200: DataStreamsResponseSchema, }, }, - getDataStreamsHandler(dataUsageContext) - ); - } + }, + getDataStreamsHandler(dataUsageService) + ); }; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts index d061aa14417df..bc8c5e898c35e 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts @@ -6,8 +6,9 @@ */ import { type ElasticsearchClient, RequestHandler } from '@kbn/core/server'; -import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; +import { DataUsageRequestHandlerContext } from '../../types'; import { errorHandler } from '../error_handler'; +import { DataUsageService } from '../../services'; export interface MeteringStats { name: string; @@ -27,9 +28,9 @@ const getMeteringStats = (client: ElasticsearchClient) => { }; export const getDataStreamsHandler = ( - dataUsageContext: DataUsageContext + dataUsageService: DataUsageService ): RequestHandler => { - const logger = dataUsageContext.logFactory.get('dataStreamsRoute'); + const logger = dataUsageService.getLogger('dataStreamsRoute'); return async (context, _, response) => { logger.debug('Retrieving user data streams'); diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts index 0013102f697fb..eeb7b44413649 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -7,33 +7,32 @@ import { UsageMetricsRequestSchema, UsageMetricsResponseSchema } from '../../../common/rest_types'; import { DATA_USAGE_METRICS_API_ROUTE } from '../../../common'; -import { DataUsageContext, DataUsageRouter } from '../../types'; +import { DataUsageRouter } from '../../types'; +import { DataUsageService } from '../../services'; import { getUsageMetricsHandler } from './usage_metrics_handler'; export const registerUsageMetricsRoute = ( router: DataUsageRouter, - dataUsageContext: DataUsageContext + dataUsageService: DataUsageService ) => { - if (dataUsageContext.serverConfig.enabled) { - router.versioned - .post({ - access: 'internal', - path: DATA_USAGE_METRICS_API_ROUTE, - }) - .addVersion( - { - version: '1', - validate: { - request: { - body: UsageMetricsRequestSchema, - }, - response: { - 200: UsageMetricsResponseSchema, - }, + router.versioned + .post({ + access: 'internal', + path: DATA_USAGE_METRICS_API_ROUTE, + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: UsageMetricsRequestSchema, + }, + response: { + 200: UsageMetricsResponseSchema, }, }, - getUsageMetricsHandler(dataUsageContext) - ); - } + }, + getUsageMetricsHandler(dataUsageService) + ); }; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 6c188662e2237..93b31033fc4fb 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -9,12 +9,12 @@ import { RequestHandler } from '@kbn/core/server'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; import { MetricTypes, - UsageMetricsAutoOpsResponseSchema, UsageMetricsAutoOpsResponseSchemaBody, UsageMetricsRequestBody, UsageMetricsResponseSchemaBody, } from '../../../common/rest_types'; -import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; +import { DataUsageRequestHandlerContext } from '../../types'; +import { DataUsageService } from '../../services'; import { errorHandler } from '../error_handler'; import { CustomHttpRequestError } from '../../utils'; @@ -23,9 +23,9 @@ const formatStringParams = (value: T | T[]): T[] | MetricTypes typeof value === 'string' ? [value] : value; export const getUsageMetricsHandler = ( - dataUsageContext: DataUsageContext + dataUsageService: DataUsageService ): RequestHandler => { - const logger = dataUsageContext.logFactory.get('usageMetricsRoute'); + const logger = dataUsageService.getLogger('usageMetricsRoute'); return async (context, request, response) => { try { @@ -50,8 +50,7 @@ export const getUsageMetricsHandler = ( name: requestDsNames, expand_wildcards: 'all', }); - - const metrics = await fetchMetricsFromAutoOps({ + const metrics = await dataUsageService.getMetrics({ from, to, metricTypes: formatStringParams(metricTypes) as MetricTypes[], @@ -70,160 +69,6 @@ export const getUsageMetricsHandler = ( }; }; -const fetchMetricsFromAutoOps = async ({ - from, - to, - metricTypes, - dataStreams, -}: { - from: string; - to: string; - metricTypes: MetricTypes[]; - dataStreams: string[]; -}) => { - // TODO: fetch data from autoOps using userDsNames - /* - const response = await axios.post({AUTOOPS_URL}, { - from: Date.parse(from), - to: Date.parse(to), - metric_types: metricTypes, - allowed_indices: dataStreams, - }); - const { data } = response;*/ - // mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml - const mockData = { - metrics: { - ingest_rate: [ - { - name: 'metrics-apache_spark.driver-default', - data: [ - [1726858530000, 13756849], - [1726862130000, 14657904], - [1726865730000, 12798561], - [1726869330000, 13578213], - [1726872930000, 14123495], - [1726876530000, 13876548], - [1726880130000, 12894561], - [1726883730000, 14478953], - [1726887330000, 14678905], - [1726890930000, 13976547], - [1726894530000, 14568945], - [1726898130000, 13789561], - [1726901730000, 14478905], - [1726905330000, 13956423], - [1726908930000, 14598234], - ], - }, - { - name: 'logs-apm.app.adservice-default', - data: [ - [1726858530000, 12894623], - [1726862130000, 14436905], - [1726865730000, 13794805], - [1726869330000, 14048532], - [1726872930000, 14237495], - [1726876530000, 13745689], - [1726880130000, 13974562], - [1726883730000, 14234653], - [1726887330000, 14323479], - [1726890930000, 14023945], - [1726894530000, 14189673], - [1726898130000, 14247895], - [1726901730000, 14098324], - [1726905330000, 14478905], - [1726908930000, 14323894], - ], - }, - { - name: 'metrics-apm.app.aws-lambdas-default', - data: [ - [1726858530000, 12576413], - [1726862130000, 13956423], - [1726865730000, 14568945], - [1726869330000, 14234856], - [1726872930000, 14368942], - [1726876530000, 13897654], - [1726880130000, 14456989], - [1726883730000, 14568956], - [1726887330000, 13987562], - [1726890930000, 14567894], - [1726894530000, 14246789], - [1726898130000, 14567895], - [1726901730000, 14457896], - [1726905330000, 14567895], - [1726908930000, 13989456], - ], - }, - ], - storage_retained: [ - { - name: 'metrics-apache_spark.driver-default', - data: [ - [1726858530000, 12576413], - [1726862130000, 13956423], - [1726865730000, 14568945], - [1726869330000, 14234856], - [1726872930000, 14368942], - [1726876530000, 13897654], - [1726880130000, 14456989], - [1726883730000, 14568956], - [1726887330000, 13987562], - [1726890930000, 14567894], - [1726894530000, 14246789], - [1726898130000, 14567895], - [1726901730000, 14457896], - [1726905330000, 14567895], - [1726908930000, 13989456], - ], - }, - { - name: 'logs-apm.app.adservice-default', - data: [ - [1726858530000, 12894623], - [1726862130000, 14436905], - [1726865730000, 13794805], - [1726869330000, 14048532], - [1726872930000, 14237495], - [1726876530000, 13745689], - [1726880130000, 13974562], - [1726883730000, 14234653], - [1726887330000, 14323479], - [1726890930000, 14023945], - [1726894530000, 14189673], - [1726898130000, 14247895], - [1726901730000, 14098324], - [1726905330000, 14478905], - [1726908930000, 14323894], - ], - }, - { - name: 'metrics-apm.app.aws-lambdas-default', - data: [ - [1726858530000, 12576413], - [1726862130000, 13956423], - [1726865730000, 14568945], - [1726869330000, 14234856], - [1726872930000, 14368942], - [1726876530000, 13897654], - [1726880130000, 14456989], - [1726883730000, 14568956], - [1726887330000, 13987562], - [1726890930000, 14567894], - [1726894530000, 14246789], - [1726898130000, 14567895], - [1726901730000, 14457896], - [1726905330000, 14567895], - [1726908930000, 13989456], - ], - }, - ], - }, - }; - // Make sure data is what we expect - const validatedData = UsageMetricsAutoOpsResponseSchema.body().validate(mockData); - - return validatedData; -}; function transformMetricsData( data: UsageMetricsAutoOpsResponseSchemaBody ): UsageMetricsResponseSchemaBody { diff --git a/x-pack/plugins/data_usage/server/app_context.ts b/x-pack/plugins/data_usage/server/services/app_context.ts similarity index 89% rename from x-pack/plugins/data_usage/server/app_context.ts rename to x-pack/plugins/data_usage/server/services/app_context.ts index e339403e3baf5..19ce666d3b01b 100644 --- a/x-pack/plugins/data_usage/server/app_context.ts +++ b/x-pack/plugins/data_usage/server/services/app_context.ts @@ -11,10 +11,10 @@ import { kibanaPackageJson } from '@kbn/repo-info'; import type { LoggerFactory } from '@kbn/core/server'; import type { CloudSetup } from '@kbn/cloud-plugin/server'; -import { DataUsageConfigType } from './config'; -import type { DataUsageContext } from './types'; +import { DataUsageConfigType } from '../config'; +import type { DataUsageContext } from '../types'; -class AppContextService { +export class AppContextService { private config$?: Observable; private configSubject$?: BehaviorSubject; private kibanaVersion: DataUsageContext['kibanaVersion'] = kibanaPackageJson.version; @@ -23,7 +23,7 @@ class AppContextService { private cloud?: CloudSetup; private logFactory?: LoggerFactory; - public start(appContext: DataUsageContext) { + constructor(appContext: DataUsageContext) { this.cloud = appContext.cloud; this.logFactory = appContext.logFactory; this.kibanaVersion = appContext.kibanaVersion; @@ -70,5 +70,3 @@ class AppContextService { return this.kibanaInstanceId; } } - -export const appContextService = new AppContextService(); diff --git a/x-pack/plugins/data_usage/server/services/autoops_api.ts b/x-pack/plugins/data_usage/server/services/autoops_api.ts index a61815a367949..ece0ec86116f2 100644 --- a/x-pack/plugins/data_usage/server/services/autoops_api.ts +++ b/x-pack/plugins/data_usage/server/services/autoops_api.ts @@ -12,13 +12,17 @@ import apm from 'elastic-apm-node'; import type { AxiosError, AxiosRequestConfig } from 'axios'; import axios from 'axios'; import { LogMeta } from '@kbn/core/server'; -import { UsageMetricsResponseSchemaBody } from '../../common/rest_types'; -import { appContextService } from '../app_context'; +import { + UsageMetricsAutoOpsResponseSchemaBody, + UsageMetricsRequestBody, +} from '../../common/rest_types'; +import { AppContextService } from './app_context'; import { AutoOpsConfig } from '../types'; -class AutoOpsAPIService { - public async autoOpsUsageMetricsAPI(requestBody: UsageMetricsResponseSchemaBody) { - const logger = appContextService.getLogger().get(); +export class AutoOpsAPIService { + constructor(private appContextService: AppContextService) {} + public async autoOpsUsageMetricsAPI(requestBody: UsageMetricsRequestBody) { + const logger = this.appContextService.getLogger().get(); const traceId = apm.currentTransaction?.traceparent; const withRequestIdMessage = (message: string) => `${message} [Request Id: ${traceId}]`; @@ -28,7 +32,7 @@ class AutoOpsAPIService { }, }; - const autoopsConfig = appContextService.getConfig()?.autoops; + const autoopsConfig = this.appContextService.getConfig()?.autoops; if (!autoopsConfig) { logger.error('[AutoOps API] Missing autoops configuration', errorMetadata); throw new Error('missing autoops configuration'); @@ -58,9 +62,9 @@ class AutoOpsAPIService { }), }; - const cloudSetup = appContextService.getCloud(); + const cloudSetup = this.appContextService.getCloud(); if (!cloudSetup?.isServerlessEnabled) { - requestConfig.data.stack_version = appContextService.getKibanaVersion(); + requestConfig.data.stack_version = this.appContextService.getKibanaVersion(); } const requestConfigDebugStatus = this.createRequestConfigDebug(requestConfig); @@ -78,7 +82,7 @@ class AutoOpsAPIService { }, }; - const response = await axios(requestConfig).catch( + const response = await axios(requestConfig).catch( (error: Error | AxiosError) => { if (!axios.isAxiosError(error)) { logger.error( @@ -171,5 +175,3 @@ class AutoOpsAPIService { return error.cause; }; } - -export const autoopsApiService = new AutoOpsAPIService(); diff --git a/x-pack/plugins/data_usage/server/services/index.ts b/x-pack/plugins/data_usage/server/services/index.ts new file mode 100644 index 0000000000000..4026891180a78 --- /dev/null +++ b/x-pack/plugins/data_usage/server/services/index.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { AppContextService } from './app_context'; +import { AutoOpsAPIService } from './autoops_api'; +import type { DataUsageContext } from '../types'; +import { MetricTypes } from '../../common/rest_types'; + +export class DataUsageService { + private appContextService: AppContextService; + private autoOpsAPIService: AutoOpsAPIService; + + constructor(dataUsageContext: DataUsageContext) { + this.appContextService = new AppContextService(dataUsageContext); + this.autoOpsAPIService = new AutoOpsAPIService(this.appContextService); + } + + getLogger(routeName: string) { + return this.appContextService.getLogger().get(routeName); + } + async getMetrics({ + from, + to, + metricTypes, + dataStreams, + }: { + from: string; + to: string; + metricTypes: MetricTypes[]; + dataStreams: string[]; + }) { + const response = await this.autoOpsAPIService.autoOpsUsageMetricsAPI({ + from, + to, + metricTypes, + dataStreams, + }); + return response.data; + } +}