Skip to content

Commit

Permalink
use auto ops service
Browse files Browse the repository at this point in the history
  • Loading branch information
@neptunian
neptunian committed Oct 15, 2024
1 parent c0bd82b commit 4ecf0bb
Show file tree
Hide file tree
Showing 2 changed files with 150 additions and 147 deletions.
286 changes: 143 additions & 143 deletions x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ import { RequestHandler } from '@kbn/core/server';
import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types';
import {
MetricTypes,
UsageMetricsAutoOpsResponseSchema,
UsageMetricsAutoOpsResponseSchemaBody,
UsageMetricsRequestBody,
UsageMetricsResponseSchemaBody,
} from '../../../common/rest_types';
import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types';
import { AutoOpsAPIService, autoopsApiService } from '../../services/autoops_api';

import { errorHandler } from '../error_handler';
import { CustomHttpRequestError } from '../../utils';
Expand Down Expand Up @@ -54,6 +54,7 @@ export const getUsageMetricsHandler = (
to,
metricTypes: formatStringParams(metricTypes) as MetricTypes[],
dataStreams: formatStringParams(dataStreamsResponse.map((ds) => ds.name)),
autoOpsAPIService: autoopsApiService,
});

const body = transformMetricsData(metrics);
Expand All @@ -73,154 +74,23 @@ const fetchMetricsFromAutoOps = async ({
to,
metricTypes,
dataStreams,
autoOpsAPIService,
}: {
from: string;
to: string;
metricTypes: MetricTypes[];
dataStreams: string[];
autoOpsAPIService: AutoOpsAPIService;
}) => {
// TODO: fetch data from autoOps using userDsNames
/*
const response = await axios.post({AUTOOPS_URL}, {
from: Date.parse(from),
to: Date.parse(to),
metric_types: metricTypes,
allowed_indices: dataStreams,
});
const { data } = response;*/
// mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml
const mockData = {
metrics: {
ingest_rate: [
{
name: 'metrics-apache_spark.driver-default',
data: [
[1726858530000, 13756849],
[1726862130000, 14657904],
[1726865730000, 12798561],
[1726869330000, 13578213],
[1726872930000, 14123495],
[1726876530000, 13876548],
[1726880130000, 12894561],
[1726883730000, 14478953],
[1726887330000, 14678905],
[1726890930000, 13976547],
[1726894530000, 14568945],
[1726898130000, 13789561],
[1726901730000, 14478905],
[1726905330000, 13956423],
[1726908930000, 14598234],
],
},
{
name: 'logs-apm.app.adservice-default',
data: [
[1726858530000, 12894623],
[1726862130000, 14436905],
[1726865730000, 13794805],
[1726869330000, 14048532],
[1726872930000, 14237495],
[1726876530000, 13745689],
[1726880130000, 13974562],
[1726883730000, 14234653],
[1726887330000, 14323479],
[1726890930000, 14023945],
[1726894530000, 14189673],
[1726898130000, 14247895],
[1726901730000, 14098324],
[1726905330000, 14478905],
[1726908930000, 14323894],
],
},
{
name: 'metrics-apm.app.aws-lambdas-default',
data: [
[1726858530000, 12576413],
[1726862130000, 13956423],
[1726865730000, 14568945],
[1726869330000, 14234856],
[1726872930000, 14368942],
[1726876530000, 13897654],
[1726880130000, 14456989],
[1726883730000, 14568956],
[1726887330000, 13987562],
[1726890930000, 14567894],
[1726894530000, 14246789],
[1726898130000, 14567895],
[1726901730000, 14457896],
[1726905330000, 14567895],
[1726908930000, 13989456],
],
},
],
storage_retained: [
{
name: 'metrics-apache_spark.driver-default',
data: [
[1726858530000, 12576413],
[1726862130000, 13956423],
[1726865730000, 14568945],
[1726869330000, 14234856],
[1726872930000, 14368942],
[1726876530000, 13897654],
[1726880130000, 14456989],
[1726883730000, 14568956],
[1726887330000, 13987562],
[1726890930000, 14567894],
[1726894530000, 14246789],
[1726898130000, 14567895],
[1726901730000, 14457896],
[1726905330000, 14567895],
[1726908930000, 13989456],
],
},
{
name: 'logs-apm.app.adservice-default',
data: [
[1726858530000, 12894623],
[1726862130000, 14436905],
[1726865730000, 13794805],
[1726869330000, 14048532],
[1726872930000, 14237495],
[1726876530000, 13745689],
[1726880130000, 13974562],
[1726883730000, 14234653],
[1726887330000, 14323479],
[1726890930000, 14023945],
[1726894530000, 14189673],
[1726898130000, 14247895],
[1726901730000, 14098324],
[1726905330000, 14478905],
[1726908930000, 14323894],
],
},
{
name: 'metrics-apm.app.aws-lambdas-default',
data: [
[1726858530000, 12576413],
[1726862130000, 13956423],
[1726865730000, 14568945],
[1726869330000, 14234856],
[1726872930000, 14368942],
[1726876530000, 13897654],
[1726880130000, 14456989],
[1726883730000, 14568956],
[1726887330000, 13987562],
[1726890930000, 14567894],
[1726894530000, 14246789],
[1726898130000, 14567895],
[1726901730000, 14457896],
[1726905330000, 14567895],
[1726908930000, 13989456],
],
},
],
},
};
// Make sure data is what we expect
const validatedData = UsageMetricsAutoOpsResponseSchema.body().validate(mockData);

return validatedData;
// return mockData;
// Call AutoOpsAPIService to fetch metrics
const response = await autoOpsAPIService?.autoOpsUsageMetricsAPI({
from,
to,
metricTypes,
dataStreams,
});
return response.data;
};
function transformMetricsData(
data: UsageMetricsAutoOpsResponseSchemaBody
Expand All @@ -240,3 +110,133 @@ function transformMetricsData(
),
};
}

// mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml
const mockData = {
metrics: {
ingest_rate: [
{
name: 'metrics-apache_spark.driver-default',
data: [
[1726858530000, 13756849],
[1726862130000, 14657904],
[1726865730000, 12798561],
[1726869330000, 13578213],
[1726872930000, 14123495],
[1726876530000, 13876548],
[1726880130000, 12894561],
[1726883730000, 14478953],
[1726887330000, 14678905],
[1726890930000, 13976547],
[1726894530000, 14568945],
[1726898130000, 13789561],
[1726901730000, 14478905],
[1726905330000, 13956423],
[1726908930000, 14598234],
],
},
{
name: 'logs-apm.app.adservice-default',
data: [
[1726858530000, 12894623],
[1726862130000, 14436905],
[1726865730000, 13794805],
[1726869330000, 14048532],
[1726872930000, 14237495],
[1726876530000, 13745689],
[1726880130000, 13974562],
[1726883730000, 14234653],
[1726887330000, 14323479],
[1726890930000, 14023945],
[1726894530000, 14189673],
[1726898130000, 14247895],
[1726901730000, 14098324],
[1726905330000, 14478905],
[1726908930000, 14323894],
],
},
{
name: 'metrics-apm.app.aws-lambdas-default',
data: [
[1726858530000, 12576413],
[1726862130000, 13956423],
[1726865730000, 14568945],
[1726869330000, 14234856],
[1726872930000, 14368942],
[1726876530000, 13897654],
[1726880130000, 14456989],
[1726883730000, 14568956],
[1726887330000, 13987562],
[1726890930000, 14567894],
[1726894530000, 14246789],
[1726898130000, 14567895],
[1726901730000, 14457896],
[1726905330000, 14567895],
[1726908930000, 13989456],
],
},
],
storage_retained: [
{
name: 'metrics-apache_spark.driver-default',
data: [
[1726858530000, 12576413],
[1726862130000, 13956423],
[1726865730000, 14568945],
[1726869330000, 14234856],
[1726872930000, 14368942],
[1726876530000, 13897654],
[1726880130000, 14456989],
[1726883730000, 14568956],
[1726887330000, 13987562],
[1726890930000, 14567894],
[1726894530000, 14246789],
[1726898130000, 14567895],
[1726901730000, 14457896],
[1726905330000, 14567895],
[1726908930000, 13989456],
],
},
{
name: 'logs-apm.app.adservice-default',
data: [
[1726858530000, 12894623],
[1726862130000, 14436905],
[1726865730000, 13794805],
[1726869330000, 14048532],
[1726872930000, 14237495],
[1726876530000, 13745689],
[1726880130000, 13974562],
[1726883730000, 14234653],
[1726887330000, 14323479],
[1726890930000, 14023945],
[1726894530000, 14189673],
[1726898130000, 14247895],
[1726901730000, 14098324],
[1726905330000, 14478905],
[1726908930000, 14323894],
],
},
{
name: 'metrics-apm.app.aws-lambdas-default',
data: [
[1726858530000, 12576413],
[1726862130000, 13956423],
[1726865730000, 14568945],
[1726869330000, 14234856],
[1726872930000, 14368942],
[1726876530000, 13897654],
[1726880130000, 14456989],
[1726883730000, 14568956],
[1726887330000, 13987562],
[1726890930000, 14567894],
[1726894530000, 14246789],
[1726898130000, 14567895],
[1726901730000, 14457896],
[1726905330000, 14567895],
[1726908930000, 13989456],
],
},
],
},
};
11 changes: 7 additions & 4 deletions x-pack/plugins/data_usage/server/services/autoops_api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,15 @@ import apm from 'elastic-apm-node';
import type { AxiosError, AxiosRequestConfig } from 'axios';
import axios from 'axios';
import { LogMeta } from '@kbn/core/server';
import { UsageMetricsResponseSchemaBody } from '../../common/rest_types';
import {
UsageMetricsAutoOpsResponseSchemaBody,
UsageMetricsRequestBody,
} from '../../common/rest_types';
import { appContextService } from '../app_context';
import { AutoOpsConfig } from '../types';

class AutoOpsAPIService {
public async autoOpsUsageMetricsAPI(requestBody: UsageMetricsResponseSchemaBody) {
export class AutoOpsAPIService {
public async autoOpsUsageMetricsAPI(requestBody: UsageMetricsRequestBody) {
const logger = appContextService.getLogger().get();
const traceId = apm.currentTransaction?.traceparent;
const withRequestIdMessage = (message: string) => `${message} [Request Id: ${traceId}]`;
Expand Down Expand Up @@ -78,7 +81,7 @@ class AutoOpsAPIService {
},
};

const response = await axios<UsageMetricsResponseSchemaBody>(requestConfig).catch(
const response = await axios<UsageMetricsAutoOpsResponseSchemaBody>(requestConfig).catch(
(error: Error | AxiosError) => {
if (!axios.isAxiosError(error)) {
logger.error(
Expand Down

0 comments on commit 4ecf0bb

Please sign in to comment.