From 4ecf0bbd9b729366593ffeb706aa0b649be5990e Mon Sep 17 00:00:00 2001 From: neptunian Date: Tue, 15 Oct 2024 09:34:29 -0400 Subject: [PATCH] use auto ops service --- .../routes/internal/usage_metrics_handler.ts | 286 +++++++++--------- .../data_usage/server/services/autoops_api.ts | 11 +- 2 files changed, 150 insertions(+), 147 deletions(-) diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 2b68dc3d37a6..df5152d07c29 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -9,12 +9,12 @@ import { RequestHandler } from '@kbn/core/server'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; import { MetricTypes, - UsageMetricsAutoOpsResponseSchema, UsageMetricsAutoOpsResponseSchemaBody, UsageMetricsRequestBody, UsageMetricsResponseSchemaBody, } from '../../../common/rest_types'; import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; +import { AutoOpsAPIService, autoopsApiService } from '../../services/autoops_api'; import { errorHandler } from '../error_handler'; import { CustomHttpRequestError } from '../../utils'; @@ -54,6 +54,7 @@ export const getUsageMetricsHandler = ( to, metricTypes: formatStringParams(metricTypes) as MetricTypes[], dataStreams: formatStringParams(dataStreamsResponse.map((ds) => ds.name)), + autoOpsAPIService: autoopsApiService, }); const body = transformMetricsData(metrics); @@ -73,154 +74,23 @@ const fetchMetricsFromAutoOps = async ({ to, metricTypes, dataStreams, + autoOpsAPIService, }: { from: string; to: string; metricTypes: MetricTypes[]; dataStreams: string[]; + autoOpsAPIService: AutoOpsAPIService; }) => { - // TODO: fetch data from autoOps using userDsNames - /* - const response = await axios.post({AUTOOPS_URL}, { - from: Date.parse(from), - to: Date.parse(to), - metric_types: metricTypes, - allowed_indices: dataStreams, - }); - const { data } = response;*/ - // mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml - const mockData = { - metrics: { - ingest_rate: [ - { - name: 'metrics-apache_spark.driver-default', - data: [ - [1726858530000, 13756849], - [1726862130000, 14657904], - [1726865730000, 12798561], - [1726869330000, 13578213], - [1726872930000, 14123495], - [1726876530000, 13876548], - [1726880130000, 12894561], - [1726883730000, 14478953], - [1726887330000, 14678905], - [1726890930000, 13976547], - [1726894530000, 14568945], - [1726898130000, 13789561], - [1726901730000, 14478905], - [1726905330000, 13956423], - [1726908930000, 14598234], - ], - }, - { - name: 'logs-apm.app.adservice-default', - data: [ - [1726858530000, 12894623], - [1726862130000, 14436905], - [1726865730000, 13794805], - [1726869330000, 14048532], - [1726872930000, 14237495], - [1726876530000, 13745689], - [1726880130000, 13974562], - [1726883730000, 14234653], - [1726887330000, 14323479], - [1726890930000, 14023945], - [1726894530000, 14189673], - [1726898130000, 14247895], - [1726901730000, 14098324], - [1726905330000, 14478905], - [1726908930000, 14323894], - ], - }, - { - name: 'metrics-apm.app.aws-lambdas-default', - data: [ - [1726858530000, 12576413], - [1726862130000, 13956423], - [1726865730000, 14568945], - [1726869330000, 14234856], - [1726872930000, 14368942], - [1726876530000, 13897654], - [1726880130000, 14456989], - [1726883730000, 14568956], - [1726887330000, 13987562], - [1726890930000, 14567894], - [1726894530000, 14246789], - [1726898130000, 14567895], - [1726901730000, 14457896], - [1726905330000, 14567895], - [1726908930000, 13989456], - ], - }, - ], - storage_retained: [ - { - name: 'metrics-apache_spark.driver-default', - data: [ - [1726858530000, 12576413], - [1726862130000, 13956423], - [1726865730000, 14568945], - [1726869330000, 14234856], - [1726872930000, 14368942], - [1726876530000, 13897654], - [1726880130000, 14456989], - [1726883730000, 14568956], - [1726887330000, 13987562], - [1726890930000, 14567894], - [1726894530000, 14246789], - [1726898130000, 14567895], - [1726901730000, 14457896], - [1726905330000, 14567895], - [1726908930000, 13989456], - ], - }, - { - name: 'logs-apm.app.adservice-default', - data: [ - [1726858530000, 12894623], - [1726862130000, 14436905], - [1726865730000, 13794805], - [1726869330000, 14048532], - [1726872930000, 14237495], - [1726876530000, 13745689], - [1726880130000, 13974562], - [1726883730000, 14234653], - [1726887330000, 14323479], - [1726890930000, 14023945], - [1726894530000, 14189673], - [1726898130000, 14247895], - [1726901730000, 14098324], - [1726905330000, 14478905], - [1726908930000, 14323894], - ], - }, - { - name: 'metrics-apm.app.aws-lambdas-default', - data: [ - [1726858530000, 12576413], - [1726862130000, 13956423], - [1726865730000, 14568945], - [1726869330000, 14234856], - [1726872930000, 14368942], - [1726876530000, 13897654], - [1726880130000, 14456989], - [1726883730000, 14568956], - [1726887330000, 13987562], - [1726890930000, 14567894], - [1726894530000, 14246789], - [1726898130000, 14567895], - [1726901730000, 14457896], - [1726905330000, 14567895], - [1726908930000, 13989456], - ], - }, - ], - }, - }; - // Make sure data is what we expect - const validatedData = UsageMetricsAutoOpsResponseSchema.body().validate(mockData); - - return validatedData; + // return mockData; + // Call AutoOpsAPIService to fetch metrics + const response = await autoOpsAPIService?.autoOpsUsageMetricsAPI({ + from, + to, + metricTypes, + dataStreams, + }); + return response.data; }; function transformMetricsData( data: UsageMetricsAutoOpsResponseSchemaBody @@ -240,3 +110,133 @@ function transformMetricsData( ), }; } + +// mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml +const mockData = { + metrics: { + ingest_rate: [ + { + name: 'metrics-apache_spark.driver-default', + data: [ + [1726858530000, 13756849], + [1726862130000, 14657904], + [1726865730000, 12798561], + [1726869330000, 13578213], + [1726872930000, 14123495], + [1726876530000, 13876548], + [1726880130000, 12894561], + [1726883730000, 14478953], + [1726887330000, 14678905], + [1726890930000, 13976547], + [1726894530000, 14568945], + [1726898130000, 13789561], + [1726901730000, 14478905], + [1726905330000, 13956423], + [1726908930000, 14598234], + ], + }, + { + name: 'logs-apm.app.adservice-default', + data: [ + [1726858530000, 12894623], + [1726862130000, 14436905], + [1726865730000, 13794805], + [1726869330000, 14048532], + [1726872930000, 14237495], + [1726876530000, 13745689], + [1726880130000, 13974562], + [1726883730000, 14234653], + [1726887330000, 14323479], + [1726890930000, 14023945], + [1726894530000, 14189673], + [1726898130000, 14247895], + [1726901730000, 14098324], + [1726905330000, 14478905], + [1726908930000, 14323894], + ], + }, + { + name: 'metrics-apm.app.aws-lambdas-default', + data: [ + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], + ], + }, + ], + storage_retained: [ + { + name: 'metrics-apache_spark.driver-default', + data: [ + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], + ], + }, + { + name: 'logs-apm.app.adservice-default', + data: [ + [1726858530000, 12894623], + [1726862130000, 14436905], + [1726865730000, 13794805], + [1726869330000, 14048532], + [1726872930000, 14237495], + [1726876530000, 13745689], + [1726880130000, 13974562], + [1726883730000, 14234653], + [1726887330000, 14323479], + [1726890930000, 14023945], + [1726894530000, 14189673], + [1726898130000, 14247895], + [1726901730000, 14098324], + [1726905330000, 14478905], + [1726908930000, 14323894], + ], + }, + { + name: 'metrics-apm.app.aws-lambdas-default', + data: [ + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], + ], + }, + ], + }, +}; diff --git a/x-pack/plugins/data_usage/server/services/autoops_api.ts b/x-pack/plugins/data_usage/server/services/autoops_api.ts index a61815a36794..04c4dd900953 100644 --- a/x-pack/plugins/data_usage/server/services/autoops_api.ts +++ b/x-pack/plugins/data_usage/server/services/autoops_api.ts @@ -12,12 +12,15 @@ import apm from 'elastic-apm-node'; import type { AxiosError, AxiosRequestConfig } from 'axios'; import axios from 'axios'; import { LogMeta } from '@kbn/core/server'; -import { UsageMetricsResponseSchemaBody } from '../../common/rest_types'; +import { + UsageMetricsAutoOpsResponseSchemaBody, + UsageMetricsRequestBody, +} from '../../common/rest_types'; import { appContextService } from '../app_context'; import { AutoOpsConfig } from '../types'; -class AutoOpsAPIService { - public async autoOpsUsageMetricsAPI(requestBody: UsageMetricsResponseSchemaBody) { +export class AutoOpsAPIService { + public async autoOpsUsageMetricsAPI(requestBody: UsageMetricsRequestBody) { const logger = appContextService.getLogger().get(); const traceId = apm.currentTransaction?.traceparent; const withRequestIdMessage = (message: string) => `${message} [Request Id: ${traceId}]`; @@ -78,7 +81,7 @@ class AutoOpsAPIService { }, }; - const response = await axios(requestConfig).catch( + const response = await axios(requestConfig).catch( (error: Error | AxiosError) => { if (!axios.isAxiosError(error)) { logger.error(