Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Data Usage] updates to metrics api #195640

Merged
merged 7 commits into from
Oct 10, 2024
Merged

[Data Usage] updates to metrics api #195640

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
08b6a92
process autoops mock data
neptunian Oct 9, 2024
1163d1f
make data_streams required for metrics api
neptunian Oct 9, 2024
2efd672
fix tests
neptunian Oct 10, 2024
5746492
remove object
neptunian Oct 10, 2024
a574727
change metrics endpoint from GET to POST and update tests
neptunian Oct 10, 2024
ba52d99
Merge branch 'main' into data-usage-process-data-autoops
neptunian Oct 10, 2024
5ffe666
Merge branch 'main' into data-usage-process-data-autoops
neptunian Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 40 additions & 19 deletions x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export const UsageMetricsRequestSchema = {
minSize: 1,
validate: (values) => {
if (values.map((v) => v.trim()).some((v) => !v.length)) {
return '[metricTypes] list can not contain empty values';
return '[metricTypes] list cannot contain empty values';
} else if (values.map((v) => v.trim()).some((v) => !isValidMetricType(v))) {
return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`;
}
Expand All @@ -62,22 +62,14 @@ export const UsageMetricsRequestSchema = {
},
}),
]),
dataStreams: schema.maybe(
schema.oneOf([
schema.arrayOf(schema.string(), {
minSize: 1,
validate: (values) => {
if (values.map((v) => v.trim()).some((v) => !v.length)) {
return '[dataStreams] list can not contain empty values';
}
},
}),
schema.string({
validate: (v) =>
v.trim().length ? undefined : '[dataStreams] must have at least one value',
}),
])
),
dataStreams: schema.arrayOf(schema.string(), {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This schema has to be a oneOf(string, array). It should work with api/path?dataStreams=xyz and with api/path?dataStreams=xyz&dataStreams=pqr. The first one is a string and the second one is an array. So you can use the previous schema def without the maybe wrapper.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right. I think in my head I was thinking this should be a POST request and we could simplify it and metricTypes to be arrays only. Given they can pass in as many data streams as they want I think we should probably switch to a POST as I can see this URL getting very large. wdyt?

minSize: 1,
validate: (values) => {
if (values.map((v) => v.trim()).some((v) => !v.length)) {
return '[dataStreams] list cannot contain empty values';
}
},
}),
}),
};

Expand All @@ -92,11 +84,40 @@ export const UsageMetricsResponseSchema = {
schema.object({
name: schema.string(),
data: schema.arrayOf(
schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 }) // Each data point is an array of 2 numbers
schema.object({
x: schema.number(),
y: schema.number(),
})
),
})
)
),
}),
};
export type UsageMetricsResponseSchemaBody = TypeOf<typeof UsageMetricsResponseSchema.body>;
export type UsageMetricsResponseSchemaBody = Omit<
TypeOf<typeof UsageMetricsResponseSchema.body>,
'metrics'
> & {
metrics: Partial<Record<MetricTypes, MetricSeries[]>>;
};
export type MetricSeries = TypeOf<
typeof UsageMetricsResponseSchema.body
>['metrics'][MetricTypes][number];

export const UsageMetricsAutoOpsResponseSchema = {
body: () =>
schema.object({
metrics: schema.recordOf(
metricTypesSchema,
schema.arrayOf(
schema.object({
name: schema.string(),
data: schema.arrayOf(schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 })),
})
)
),
}),
};
export type UsageMetricsAutoOpsResponseSchemaBody = TypeOf<
typeof UsageMetricsAutoOpsResponseSchema.body
>;
13 changes: 7 additions & 6 deletions x-pack/plugins/data_usage/public/app/components/chart_panel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ import {
} from '@elastic/charts';
import { i18n } from '@kbn/i18n';
import { LegendAction } from './legend_action';
import { MetricTypes } from '../../../common/rest_types';
import { MetricSeries } from '../types';
import { MetricTypes, MetricSeries } from '../../../common/rest_types';

// TODO: Remove this when we have a title for each metric type
type ChartKey = Extract<MetricTypes, 'ingest_rate' | 'storage_retained'>;
Expand Down Expand Up @@ -50,7 +49,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
}) => {
const theme = useEuiTheme();

const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d[0]));
const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d.x));

const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)];

Expand All @@ -72,6 +71,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
},
[idx, popoverOpen, togglePopover]
);

return (
<EuiFlexItem grow={false} key={metricType}>
<EuiPanel hasShadow={false} hasBorder={true}>
Expand All @@ -94,9 +94,9 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
data={stream.data}
xScaleType={ScaleType.Time}
yScaleType={ScaleType.Linear}
xAccessor={0} // x is the first element in the tuple
yAccessors={[1]} // y is the second element in the tuple
stackAccessors={[0]}
xAccessor="x"
yAccessors={['y']}
stackAccessors={['x']}
/>
))}

Expand All @@ -118,6 +118,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
</EuiFlexItem>
);
};

const formatBytes = (bytes: number) => {
return numeral(bytes).format('0.0 b');
};
4 changes: 2 additions & 2 deletions x-pack/plugins/data_usage/public/app/components/charts.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
*/
import React, { useCallback, useState } from 'react';
import { EuiFlexGroup } from '@elastic/eui';
import { MetricsResponse } from '../types';
import { MetricTypes } from '../../../common/rest_types';
import { ChartPanel } from './chart_panel';
import { UsageMetricsResponseSchemaBody } from '../../../common/rest_types';
interface ChartsProps {
data: MetricsResponse;
data: UsageMetricsResponseSchemaBody;
}

export const Charts: React.FC<ChartsProps> = ({ data }) => {
Expand Down
9 changes: 6 additions & 3 deletions x-pack/plugins/data_usage/public/app/data_usage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import { PLUGIN_NAME } from '../../common';
import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics';
import { DEFAULT_DATE_RANGE_OPTIONS, useDateRangePicker } from './hooks/use_date_picker';
import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params';
import { MetricsResponse } from './types';

export const DataUsage = () => {
const {
Expand All @@ -44,7 +43,11 @@ export const DataUsage = () => {

const [queryParams, setQueryParams] = useState<UsageMetricsRequestSchemaQueryParams>({
metricTypes: ['storage_retained', 'ingest_rate'],
dataStreams: [],
// TODO: Replace with data streams from /data_streams api
dataStreams: [
'.alerts-ml.anomaly-detection-health.alerts-default',
'.alerts-stack.alerts-default',
],
Comment on lines +47 to +50
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Q: With respect to UX, should we use all the datastreams from the API here and have them all selected when we land on the page even if the list is very long, say 100 or more?

Copy link
Contributor Author

@neptunian neptunian Oct 10, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably a question for a PM. Personally, I would say the "top" 10 would suffice. Otherwise we are potentially making the initial experience very slow unnecessarily.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that is my concern. Also since we're pulling the datastreams list from another API (/_metering/stats) that doesn't allow for paging, I'm not sure how we're going to handle cases when users have a huge list of data streams.

Copy link
Contributor Author

@neptunian neptunian Oct 10, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At the moment I think the plan is to not handle it and just display them all in the dropdown but only select the largest by storage size of 10. The dropdown with all the streams will ideally scroll and not go off the page.

from: DEFAULT_DATE_RANGE_OPTIONS.startDate,
to: DEFAULT_DATE_RANGE_OPTIONS.endDate,
});
Expand Down Expand Up @@ -140,7 +143,7 @@ export const DataUsage = () => {
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size="l" />
{isFetched && data ? <Charts data={data as MetricsResponse} /> : <EuiLoadingElastic />}
{isFetched && data ? <Charts data={data} /> : <EuiLoadingElastic />}
</EuiPageSection>
</>
);
Expand Down
24 changes: 0 additions & 24 deletions x-pack/plugins/data_usage/public/app/types.ts
View file
Open in desktop

This file was deleted.

57 changes: 30 additions & 27 deletions x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ import { RequestHandler } from '@kbn/core/server';
import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types';
import {
MetricTypes,
UsageMetricsAutoOpsResponseSchema,
UsageMetricsAutoOpsResponseSchemaBody,
UsageMetricsRequestSchemaQueryParams,
UsageMetricsResponseSchema,
UsageMetricsResponseSchemaBody,
} from '../../../common/rest_types';
import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types';

Expand All @@ -34,44 +36,27 @@ export const getUsageMetricsHandler = (
const core = await context.core;
const esClient = core.elasticsearch.client.asCurrentUser;

// @ts-ignore
const { from, to, metricTypes, dataStreams: dsNames, size } = request.query;
const { from, to, metricTypes, dataStreams: requestDsNames } = request.query;
logger.debug(`Retrieving usage metrics`);

const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse =
await esClient.indices.getDataStream({
name: '*',
name: requestDsNames,
expand_wildcards: 'all',
});
ashokaditya marked this conversation as resolved.
Show resolved Hide resolved

const hasDataStreams = dataStreamsResponse.length > 0;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

esClient.indices.getDataStream will throw an error if a requested data stream is not found. since data streams are required, the logic below would not be met.

let userDsNames: string[] = [];

if (dsNames?.length) {
userDsNames = typeof dsNames === 'string' ? [dsNames] : dsNames;
} else if (!userDsNames.length && hasDataStreams) {
userDsNames = dataStreamsResponse.map((ds) => ds.name);
}

// If no data streams are found, return an empty response
if (!userDsNames.length) {
return response.ok({
body: {
metrics: {},
},
});
}

const metrics = await fetchMetricsFromAutoOps({
from,
to,
metricTypes: formatStringParams(metricTypes) as MetricTypes[],
dataStreams: formatStringParams(userDsNames),
dataStreams: formatStringParams(dataStreamsResponse.map((ds) => ds.name)),
ashokaditya marked this conversation as resolved.
Show resolved Hide resolved
});

const processedMetrics = transformMetricsData(metrics);

return response.ok({
body: {
metrics,
...processedMetrics,
neptunian marked this conversation as resolved.
Show resolved Hide resolved
},
});
} catch (error) {
Expand All @@ -94,7 +79,7 @@ const fetchMetricsFromAutoOps = async ({
}) => {
// TODO: fetch data from autoOps using userDsNames
/*
const response = await axios.post('https://api.auto-ops.{region}.{csp}.cloud.elastic.co/monitoring/serverless/v1/projects/{project_id}/metrics', {
const response = await axios.post({AUTOOPS_URL}, {
from: Date.parse(from),
to: Date.parse(to),
metric_types: metricTypes,
Expand Down Expand Up @@ -231,7 +216,25 @@ const fetchMetricsFromAutoOps = async ({
},
};
// Make sure data is what we expect
const validatedData = UsageMetricsResponseSchema.body().validate(mockData);
const validatedData = UsageMetricsAutoOpsResponseSchema.body().validate(mockData);

return validatedData.metrics;
return validatedData;
};
function transformMetricsData(
data: UsageMetricsAutoOpsResponseSchemaBody
): UsageMetricsResponseSchemaBody {
return {
metrics: Object.fromEntries(
Object.entries(data.metrics).map(([metricType, series]) => [
metricType,
series.map((metricSeries) => ({
name: metricSeries.name,
data: (metricSeries.data as Array<[number, number]>).map(([timestamp, value]) => ({
x: timestamp,
y: value,
})),
})),
])
),
};
}