From 78baf01ca6d32f19229856766e99f30591cfb5e7 Mon Sep 17 00:00:00 2001 From: neptunian Date: Tue, 24 Sep 2024 15:34:35 -0400 Subject: [PATCH 01/53] add ui components to data_usage and mock data --- x-pack/plugins/data_usage/common/types.ts | 9 + x-pack/plugins/data_usage/kibana.jsonc | 3 + .../public/app/components/charts.tsx | 87 +++++++++ .../public/app/components/date_picker.tsx | 52 +++++ .../data_usage/public/app/data_usage.tsx | 181 ++++++++++++++++++ .../public/app/hooks/use_date_picker.tsx | 29 +++ x-pack/plugins/data_usage/public/app/types.ts | 25 +++ .../plugins/data_usage/public/application.tsx | 3 +- .../public/utils/use_breadcrumbs.tsx | 28 +++ 9 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/data_usage/common/types.ts create mode 100644 x-pack/plugins/data_usage/public/app/components/charts.tsx create mode 100644 x-pack/plugins/data_usage/public/app/components/date_picker.tsx create mode 100644 x-pack/plugins/data_usage/public/app/data_usage.tsx create mode 100644 x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx create mode 100644 x-pack/plugins/data_usage/public/app/types.ts create mode 100644 x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx diff --git a/x-pack/plugins/data_usage/common/types.ts b/x-pack/plugins/data_usage/common/types.ts new file mode 100644 index 0000000000000..d80bae2458d09 --- /dev/null +++ b/x-pack/plugins/data_usage/common/types.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// temporary type until agreed on +export type MetricKey = 'ingestedMax' | 'retainedMax'; diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 9b0f2d193925e..82411d169ef0c 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -12,5 +12,8 @@ "requiredBundles": [ "kibanaReact", ], + "extraPublicDirs": [ + "common", + ] } } diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx new file mode 100644 index 0000000000000..f4d6cb3883245 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; +import { Chart, Axis, BarSeries, Settings, ScaleType, niceTimeFormatter } from '@elastic/charts'; +import numeral from '@elastic/numeral'; +import { i18n } from '@kbn/i18n'; +import { MetricsResponse } from '../types'; +import { MetricKey } from '../../../common/types'; +interface ChartsProps { + data: MetricsResponse; +} +const formatBytes = (bytes: number) => { + return numeral(bytes).format('0.0 b'); +}; + +export const chartKeyToTitleMap: Record = { + ingestedMax: i18n.translate('xpack.dataUsage.charts.ingestedMax', { + defaultMessage: 'Data Ingested', + }), + retainedMax: i18n.translate('xpack.dataUsage.charts.retainedMax', { + defaultMessage: 'Data Retained in Storage', + }), +}; + +export const Charts: React.FC = ({ data }) => { + return ( + + {data.charts.map((chart, idx) => { + const chartTimestamps = chart.series.flatMap((series) => series.data.map((d) => d.x)); + const minTimestamp = Math.min(...chartTimestamps); + const maxTimestamp = Math.max(...chartTimestamps); + const tickFormat = niceTimeFormatter([minTimestamp, maxTimestamp]); + + return ( + + +
+ +
{chartKeyToTitleMap[chart.key] || chart.key}
+ + + + {chart.series.map((stream, streamIdx) => ( + [point.x, point.y])} + xScaleType={ScaleType.Time} + yScaleType={ScaleType.Linear} + xAccessor={0} + yAccessors={[1]} + stackAccessors={[0]} + /> + ))} + + + + formatBytes(d)} + /> + +
+ + + ); + })} + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx new file mode 100644 index 0000000000000..f95af6d6bfc43 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSuperDatePicker } from '@elastic/eui'; +import { useDatePickerContext } from '../hooks/use_date_picker'; + +export const DatePicker: React.FC = () => { + const { + startDate, + endDate, + isAutoRefreshActive, + refreshInterval, + setStartDate, + setEndDate, + setIsAutoRefreshActive, + setRefreshInterval, + } = useDatePickerContext(); + + const onTimeChange = ({ start, end }: { start: string; end: string }) => { + setStartDate(start); + setEndDate(end); + // Trigger API call or data refresh here + }; + + const onRefreshChange = ({ + isPaused, + refreshInterval: onRefreshRefreshInterval, + }: { + isPaused: boolean; + refreshInterval: number; + }) => { + setIsAutoRefreshActive(!isPaused); + setRefreshInterval(onRefreshRefreshInterval); + }; + + return ( + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx new file mode 100644 index 0000000000000..3c80854bf9cfe --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { Charts } from './components/charts'; +import { DatePicker } from './components/date_picker'; +import { MetricsResponse } from './types'; +import { useBreadcrumbs } from '../utils/use_breadcrumbs'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; +import { PLUGIN_NAME } from '../../common'; +import { DatePickerProvider } from './hooks/use_date_picker'; + +const response: MetricsResponse = { + charts: [ + { + key: 'ingestedMax', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 13756849 }, + { x: 1726862130000, y: 14657904 }, + { x: 1726865730000, y: 12798561 }, + { x: 1726869330000, y: 13578213 }, + { x: 1726872930000, y: 14123495 }, + { x: 1726876530000, y: 13876548 }, + { x: 1726880130000, y: 12894561 }, + { x: 1726883730000, y: 14478953 }, + { x: 1726887330000, y: 14678905 }, + { x: 1726890930000, y: 13976547 }, + { x: 1726894530000, y: 14568945 }, + { x: 1726898130000, y: 13789561 }, + { x: 1726901730000, y: 14478905 }, + { x: 1726905330000, y: 13956423 }, + { x: 1726908930000, y: 14598234 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + ], + }, + { + key: 'retainedMax', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + // Repeat similar structure for more data streams... + ], + }, + ], +}; + +export const DataUsage = () => { + const { + services: { chrome, appParams }, + } = useKibanaContextForPlugin(); + + useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); + + return ( + + +

+ {i18n.translate('xpack.dataUsage.pageTitle', { + defaultMessage: 'Data usage stats', + })} +

+ + + + + + + + + ; + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx new file mode 100644 index 0000000000000..cd8ac35c98839 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState } from 'react'; +import createContainer from 'constate'; + +const useDatePicker = () => { + const [startDate, setStartDate] = useState('now-24h'); + const [endDate, setEndDate] = useState('now'); + const [isAutoRefreshActive, setIsAutoRefreshActive] = useState(false); + const [refreshInterval, setRefreshInterval] = useState(0); + + return { + startDate, + setStartDate, + endDate, + setEndDate, + isAutoRefreshActive, + setIsAutoRefreshActive, + refreshInterval, + setRefreshInterval, + }; +}; + +export const [DatePickerProvider, useDatePickerContext] = createContainer(useDatePicker); diff --git a/x-pack/plugins/data_usage/public/app/types.ts b/x-pack/plugins/data_usage/public/app/types.ts new file mode 100644 index 0000000000000..279a7e0e863ef --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { MetricKey } from '../../common/types'; + +export interface DataPoint { + x: number; + y: number; +} + +export interface Series { + streamName: string; + data: DataPoint[]; +} +export interface Chart { + key: MetricKey; + series: Series[]; +} + +export interface MetricsResponse { + charts: Chart[]; +} diff --git a/x-pack/plugins/data_usage/public/application.tsx b/x-pack/plugins/data_usage/public/application.tsx index 1e6c35c4b8f0a..a160a5742d36d 100644 --- a/x-pack/plugins/data_usage/public/application.tsx +++ b/x-pack/plugins/data_usage/public/application.tsx @@ -16,6 +16,7 @@ import { PerformanceContextProvider } from '@kbn/ebt-tools'; import { useKibanaContextForPluginProvider } from './utils/use_kibana'; import { DataUsageStartDependencies, DataUsagePublicStart } from './types'; import { PLUGIN_ID } from '../common'; +import { DataUsage } from './app/data_usage'; export const renderApp = ( core: CoreStart, @@ -51,7 +52,7 @@ const AppWithExecutionContext = ({ -
Data Usage
} /> + diff --git a/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx b/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx new file mode 100644 index 0000000000000..928ee73ad5280 --- /dev/null +++ b/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ChromeBreadcrumb, ChromeStart } from '@kbn/core-chrome-browser'; + +import { useEffect } from 'react'; +import { ManagementAppMountParams } from '@kbn/management-plugin/public'; + +export const useBreadcrumbs = ( + breadcrumbs: ChromeBreadcrumb[], + params: ManagementAppMountParams, + chromeService: ChromeStart +) => { + const { docTitle } = chromeService; + const isMultiple = breadcrumbs.length > 1; + + const docTitleValue = isMultiple ? breadcrumbs[breadcrumbs.length - 1].text : breadcrumbs[0].text; + + docTitle.change(docTitleValue as string); + + useEffect(() => { + params.setBreadcrumbs(breadcrumbs); + }, [breadcrumbs, params]); +}; From 5b0ef1e58bb8d8f659028ad7d5f2faa8c4403652 Mon Sep 17 00:00:00 2001 From: neptunian Date: Wed, 25 Sep 2024 07:55:22 -0400 Subject: [PATCH 02/53] remove deprecated extraPublicDirs --- x-pack/plugins/data_usage/kibana.jsonc | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 82411d169ef0c..9b0f2d193925e 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -12,8 +12,5 @@ "requiredBundles": [ "kibanaReact", ], - "extraPublicDirs": [ - "common", - ] } } From ec526765155d19d34897b6c0282dcacdc1f59026 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 30 Sep 2024 13:55:33 -0400 Subject: [PATCH 03/53] update icon card to stats --- packages/kbn-management/cards_navigation/src/consts.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-management/cards_navigation/src/consts.tsx b/packages/kbn-management/cards_navigation/src/consts.tsx index 16e655c5510ad..6a22b9e33d620 100644 --- a/packages/kbn-management/cards_navigation/src/consts.tsx +++ b/packages/kbn-management/cards_navigation/src/consts.tsx @@ -77,7 +77,7 @@ export const appDefinitions: Record = { description: i18n.translate('management.landing.withCardNavigation.dataUsageDescription', { defaultMessage: 'View data usage and retention.', }), - icon: 'documents', + icon: 'stats', }, [AppIds.RULES]: { From 4d4114e6956303220a2979986cc789a35ca65dae Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 30 Sep 2024 15:55:54 -0400 Subject: [PATCH 04/53] data stream action menu --- .../public/app/components/charts.tsx | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index f4d6cb3883245..3308e5b15304c 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -4,9 +4,26 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; -import { Chart, Axis, BarSeries, Settings, ScaleType, niceTimeFormatter } from '@elastic/charts'; +import React, { useState } from 'react'; +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiListGroup, + EuiListGroupItem, + EuiPanel, + EuiPopover, + EuiTitle, +} from '@elastic/eui'; +import { + Chart, + Axis, + BarSeries, + Settings, + ScaleType, + niceTimeFormatter, + LegendActionProps, +} from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; import { MetricsResponse } from '../types'; @@ -28,6 +45,11 @@ export const chartKeyToTitleMap: Record = { }; export const Charts: React.FC = ({ data }) => { + const [popoverOpen, setPopoverOpen] = useState(null); + + const togglePopover = (streamName: string) => { + setPopoverOpen(popoverOpen === streamName ? null : streamName); + }; return ( {data.charts.map((chart, idx) => { @@ -48,6 +70,47 @@ export const Charts: React.FC = ({ data }) => { showLegend={true} legendPosition="right" xDomain={{ min: minTimestamp, max: maxTimestamp }} + legendAction={({ label }: LegendActionProps) => { + const streamName = `${idx}-${label}`; + return ( + + + + togglePopover(streamName)} + /> + + + } + isOpen={popoverOpen === streamName} + closePopover={() => setPopoverOpen(null)} + anchorPosition="downRight" + > + + undefined} + /> + undefined} + /> + undefined} + /> + + + + ); + }} /> {chart.series.map((stream, streamIdx) => ( Date: Thu, 3 Oct 2024 07:06:17 -0400 Subject: [PATCH 05/53] add elasticsearch feature privilege --- x-pack/plugins/data_usage/server/plugin.ts | 15 ++++++++++++++- x-pack/plugins/data_usage/server/types.ts | 7 +++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_usage/server/plugin.ts b/x-pack/plugins/data_usage/server/plugin.ts index 8ab49d5104fff..d4309d5742528 100644 --- a/x-pack/plugins/data_usage/server/plugin.ts +++ b/x-pack/plugins/data_usage/server/plugin.ts @@ -14,6 +14,7 @@ import type { DataUsageSetupDependencies, DataUsageStartDependencies, } from './types'; +import { PLUGIN_ID } from '../common'; export class DataUsagePlugin implements @@ -28,7 +29,19 @@ export class DataUsagePlugin constructor(context: PluginInitializerContext) { this.logger = context.logger.get(); } - setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup { + setup(coreSetup: CoreSetup, { features }: DataUsageSetupDependencies): DataUsageServerSetup { + features.registerElasticsearchFeature({ + id: PLUGIN_ID, + management: { + data: [PLUGIN_ID], + }, + privileges: [ + { + requiredClusterPrivileges: ['monitor'], + ui: [], + }, + ], + }); return {}; } diff --git a/x-pack/plugins/data_usage/server/types.ts b/x-pack/plugins/data_usage/server/types.ts index 9f43ae2d3c298..fa0d561701a9d 100644 --- a/x-pack/plugins/data_usage/server/types.ts +++ b/x-pack/plugins/data_usage/server/types.ts @@ -5,10 +5,13 @@ * 2.0. */ -/* eslint-disable @typescript-eslint/no-empty-interface*/ +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; -export interface DataUsageSetupDependencies {} +export interface DataUsageSetupDependencies { + features: FeaturesPluginSetup; +} +/* eslint-disable @typescript-eslint/no-empty-interface*/ export interface DataUsageStartDependencies {} export interface DataUsageServerSetup {} From 274f6d46f0d45130294884095253aecf08103f41 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 08:57:58 -0400 Subject: [PATCH 06/53] add descriptive text and fix layout spacing --- .../data_usage/public/app/data_usage.tsx | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 3c80854bf9cfe..656e5e81ebc5d 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -6,8 +6,9 @@ */ import React from 'react'; -import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem, EuiText, EuiPageSection } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import { Charts } from './components/charts'; import { DatePicker } from './components/date_picker'; import { MetricsResponse } from './types'; @@ -169,13 +170,23 @@ export const DataUsage = () => { - - - - - - - ; + + + + + + + + + + + + + ; + ); }; From 41a584b0cdf11be7543d67851267795f7e34bff0 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 09:36:05 -0400 Subject: [PATCH 07/53] fix chart theme dark mode --- x-pack/plugins/data_usage/public/app/components/charts.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 3308e5b15304c..3e74acc685c9a 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -14,6 +14,7 @@ import { EuiPanel, EuiPopover, EuiTitle, + useEuiTheme, } from '@elastic/eui'; import { Chart, @@ -23,9 +24,12 @@ import { ScaleType, niceTimeFormatter, LegendActionProps, + DARK_THEME, + LIGHT_THEME, } from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; + import { MetricsResponse } from '../types'; import { MetricKey } from '../../../common/types'; interface ChartsProps { @@ -46,6 +50,7 @@ export const chartKeyToTitleMap: Record = { export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); + const theme = useEuiTheme(); const togglePopover = (streamName: string) => { setPopoverOpen(popoverOpen === streamName ? null : streamName); @@ -67,6 +72,7 @@ export const Charts: React.FC = ({ data }) => { Date: Thu, 3 Oct 2024 11:37:16 -0400 Subject: [PATCH 08/53] fix type --- x-pack/plugins/data_usage/public/app/data_usage.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 656e5e81ebc5d..d97376252cd1d 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -6,7 +6,14 @@ */ import React from 'react'; -import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem, EuiText, EuiPageSection } from '@elastic/eui'; +import { + EuiTitle, + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiPageSection, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { Charts } from './components/charts'; From 1fe1f9af1e9cd981d4bd7b856d4c3d132a1a3342 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 15:29:01 -0400 Subject: [PATCH 09/53] add data quality set locator --- .../observability/locators/dataset_quality.ts | 1 + .../public/app/components/charts.tsx | 59 +++++++++++++++---- .../data_usage/public/app/data_usage.tsx | 2 +- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/packages/deeplinks/observability/locators/dataset_quality.ts b/packages/deeplinks/observability/locators/dataset_quality.ts index 9a6dd85ade2d2..6ae42f3805381 100644 --- a/packages/deeplinks/observability/locators/dataset_quality.ts +++ b/packages/deeplinks/observability/locators/dataset_quality.ts @@ -27,6 +27,7 @@ type TimeRangeConfig = { // eslint-disable-next-line @typescript-eslint/consistent-type-definitions type Filters = { timeRange: TimeRangeConfig; + query?: string; }; export interface DataQualityLocatorParams extends SerializableRecord { diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 3e74acc685c9a..08f51f60f887c 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -29,9 +29,10 @@ import { } from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; - +import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { MetricsResponse } from '../types'; import { MetricKey } from '../../../common/types'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; interface ChartsProps { data: MetricsResponse; } @@ -47,10 +48,41 @@ export const chartKeyToTitleMap: Record = { defaultMessage: 'Data Retained in Storage', }), }; - +function getDatasetFromDataStream(dataStreamName: string): string | null { + const parts = dataStreamName.split('-'); + if (parts.length !== 3) { + return null; + } + return parts[1]; +} export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); const theme = useEuiTheme(); + const { + services: { + share: { + url: { locators }, + }, + application: { capabilities }, + }, + } = useKibanaContextForPlugin(); + const hasDataSetQualityFeature = capabilities.data_quality; + const onClickDataQuality = async ({ dataStreamName }: { dataStreamName: string }) => { + const dataQualityLocator = locators.get(DATA_QUALITY_LOCATOR_ID); + const locatorParams: DataQualityLocatorParams = { + filters: { + // TODO: get time range from our page state + timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + }, + }; + const dataset = getDatasetFromDataStream(dataStreamName); + if (locatorParams?.filters && dataset) { + locatorParams.filters.query = dataset; + } + if (dataQualityLocator) { + await dataQualityLocator.navigate(locatorParams); + } + }; const togglePopover = (streamName: string) => { setPopoverOpen(popoverOpen === streamName ? null : streamName); @@ -64,7 +96,7 @@ export const Charts: React.FC = ({ data }) => { const tickFormat = niceTimeFormatter([minTimestamp, maxTimestamp]); return ( - +
@@ -77,7 +109,7 @@ export const Charts: React.FC = ({ data }) => { legendPosition="right" xDomain={{ min: minTimestamp, max: maxTimestamp }} legendAction={({ label }: LegendActionProps) => { - const streamName = `${idx}-${label}`; + const uniqueStreamName = `${idx}-${label}`; return ( = ({ data }) => { togglePopover(streamName)} + onClick={() => togglePopover(uniqueStreamName)} /> } - isOpen={popoverOpen === streamName} + isOpen={popoverOpen === uniqueStreamName} closePopover={() => setPopoverOpen(null)} anchorPosition="downRight" > @@ -107,11 +139,16 @@ export const Charts: React.FC = ({ data }) => { label="Manage data stream" onClick={() => undefined} /> - undefined} - /> + {hasDataSetQualityFeature && ( + + onClickDataQuality({ + dataStreamName: label, + }) + } + /> + )} diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index d97376252cd1d..a43f18f0aa30a 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -192,7 +192,7 @@ export const DataUsage = () => { - ; + ); From 817a112fa3314045f41f10591f4106dcb9e62a2c Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 19:49:38 -0400 Subject: [PATCH 10/53] fix title --- x-pack/plugins/data_usage/public/app/data_usage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index a43f18f0aa30a..476caaa8930a7 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -172,7 +172,7 @@ export const DataUsage = () => {

{i18n.translate('xpack.dataUsage.pageTitle', { - defaultMessage: 'Data usage stats', + defaultMessage: 'Data Usage', })}

From d3e92967001f35fd3083bee16d1b09c18399e674 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 25 Sep 2024 12:17:35 +0200 Subject: [PATCH 11/53] initial internal API setup --- .../data_usage/server/common/errors.ts | 18 ++ x-pack/plugins/data_usage/server/config.ts | 13 +- x-pack/plugins/data_usage/server/index.ts | 8 +- x-pack/plugins/data_usage/server/plugin.ts | 33 +++- .../data_usage/server/routes/error_handler.ts | 40 ++++ .../data_usage/server/routes/index.tsx | 16 ++ .../server/routes/internal/index.tsx | 7 + .../server/routes/internal/usage_metrics.ts | 40 ++++ .../routes/internal/usage_metrics_handler.ts | 139 ++++++++++++++ x-pack/plugins/data_usage/server/types.ts | 19 -- .../plugins/data_usage/server/types/index.ts | 9 + .../server/types/rest_types/index.ts | 8 + .../types/rest_types/usage_metrics.test.ts | 174 ++++++++++++++++++ .../server/types/rest_types/usage_metrics.ts | 104 +++++++++++ .../plugins/data_usage/server/types/types.ts | 42 +++++ .../server/utils/custom_http_request_error.ts | 22 +++ .../plugins/data_usage/server/utils/index.ts | 9 + .../server/utils/validate_time_range.ts | 17 ++ 18 files changed, 687 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugins/data_usage/server/common/errors.ts create mode 100644 x-pack/plugins/data_usage/server/routes/error_handler.ts create mode 100644 x-pack/plugins/data_usage/server/routes/index.tsx create mode 100644 x-pack/plugins/data_usage/server/routes/internal/index.tsx create mode 100644 x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts create mode 100644 x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts delete mode 100644 x-pack/plugins/data_usage/server/types.ts create mode 100644 x-pack/plugins/data_usage/server/types/index.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/index.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts create mode 100644 x-pack/plugins/data_usage/server/types/types.ts create mode 100644 x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts create mode 100644 x-pack/plugins/data_usage/server/utils/index.ts create mode 100644 x-pack/plugins/data_usage/server/utils/validate_time_range.ts diff --git a/x-pack/plugins/data_usage/server/common/errors.ts b/x-pack/plugins/data_usage/server/common/errors.ts new file mode 100644 index 0000000000000..7a43a10108be1 --- /dev/null +++ b/x-pack/plugins/data_usage/server/common/errors.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class BaseError extends Error { + constructor(message: string, public readonly meta?: MetaType) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } + } +} diff --git a/x-pack/plugins/data_usage/server/config.ts b/x-pack/plugins/data_usage/server/config.ts index 6453cce4f4d56..bf89431f2abea 100644 --- a/x-pack/plugins/data_usage/server/config.ts +++ b/x-pack/plugins/data_usage/server/config.ts @@ -6,9 +6,18 @@ */ import { schema, type TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from '@kbn/core/server'; -export const config = schema.object({ +export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: false }), }); -export type DataUsageConfig = TypeOf; +export type DataUsageConfigType = TypeOf; + +export const createConfig = (context: PluginInitializerContext): DataUsageConfigType => { + const pluginConfig = context.config.get>(); + + return { + ...pluginConfig, + }; +}; diff --git a/x-pack/plugins/data_usage/server/index.ts b/x-pack/plugins/data_usage/server/index.ts index 3aa49a184d003..66d839303d716 100644 --- a/x-pack/plugins/data_usage/server/index.ts +++ b/x-pack/plugins/data_usage/server/index.ts @@ -9,7 +9,7 @@ import type { PluginInitializerContext, PluginConfigDescriptor, } from '@kbn/core/server'; -import { DataUsageConfig } from './config'; +import { DataUsageConfigType } from './config'; import { DataUsagePlugin } from './plugin'; import type { @@ -19,11 +19,11 @@ import type { DataUsageStartDependencies, } from './types'; -import { config as configSchema } from './config'; +import { configSchema } from './config'; export type { DataUsageServerSetup, DataUsageServerStart }; -export const config: PluginConfigDescriptor = { +export const config: PluginConfigDescriptor = { schema: configSchema, }; @@ -32,5 +32,5 @@ export const plugin: PluginInitializer< DataUsageServerStart, DataUsageSetupDependencies, DataUsageStartDependencies -> = async (pluginInitializerContext: PluginInitializerContext) => +> = async (pluginInitializerContext: PluginInitializerContext) => await new DataUsagePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/data_usage/server/plugin.ts b/x-pack/plugins/data_usage/server/plugin.ts index d4309d5742528..2beae9b22bba9 100644 --- a/x-pack/plugins/data_usage/server/plugin.ts +++ b/x-pack/plugins/data_usage/server/plugin.ts @@ -7,13 +7,16 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; -import { DataUsageConfig } from './config'; +import { DataUsageConfigType, createConfig } from './config'; import type { + DataUsageContext, + DataUsageRequestHandlerContext, DataUsageServerSetup, DataUsageServerStart, DataUsageSetupDependencies, DataUsageStartDependencies, } from './types'; +import { registerDataUsageRoutes } from './routes'; import { PLUGIN_ID } from '../common'; export class DataUsagePlugin @@ -25,12 +28,25 @@ export class DataUsagePlugin DataUsageStartDependencies > { - logger: Logger; - constructor(context: PluginInitializerContext) { + private readonly logger: Logger; + private dataUsageContext: DataUsageContext; + + constructor(context: PluginInitializerContext) { + const serverConfig = createConfig(context); + this.logger = context.logger.get(); + + this.logger.debug('data usage plugin initialized'); + this.dataUsageContext = { + logFactory: context.logger, + get serverConfig() { + return serverConfig; + }, + }; } - setup(coreSetup: CoreSetup, { features }: DataUsageSetupDependencies): DataUsageServerSetup { - features.registerElasticsearchFeature({ + setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup { + this.logger.debug('data usage plugin setup'); + pluginsSetup.features.registerElasticsearchFeature({ id: PLUGIN_ID, management: { data: [PLUGIN_ID], @@ -42,6 +58,9 @@ export class DataUsagePlugin }, ], }); + const router = coreSetup.http.createRouter(); + registerDataUsageRoutes(router, this.dataUsageContext); + return {}; } @@ -49,5 +68,7 @@ export class DataUsagePlugin return {}; } - public stop() {} + public stop() { + this.logger.debug('Stopping data usage plugin'); + } } diff --git a/x-pack/plugins/data_usage/server/routes/error_handler.ts b/x-pack/plugins/data_usage/server/routes/error_handler.ts new file mode 100644 index 0000000000000..7bd774f2e71e5 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/error_handler.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; +import { CustomHttpRequestError } from '../utils/custom_http_request_error'; +import { BaseError } from '../common/errors'; + +export class NotFoundError extends BaseError {} + +/** + * Default Endpoint Routes error handler + * @param logger + * @param res + * @param error + */ +export const errorHandler = ( + logger: Logger, + res: KibanaResponseFactory, + error: E +): IKibanaResponse => { + logger.error(error); + + if (error instanceof CustomHttpRequestError) { + return res.customError({ + statusCode: error.statusCode, + body: error, + }); + } + + if (error instanceof NotFoundError) { + return res.notFound({ body: error }); + } + + // Kibana CORE will take care of `500` errors when the handler `throw`'s, including logging the error + throw error; +}; diff --git a/x-pack/plugins/data_usage/server/routes/index.tsx b/x-pack/plugins/data_usage/server/routes/index.tsx new file mode 100644 index 0000000000000..b2d10b0ab53bc --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/index.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataUsageContext, DataUsageRouter } from '../types'; +import { registerUsageMetricsRoute } from './internal'; + +export const registerDataUsageRoutes = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + registerUsageMetricsRoute(router, dataUsageContext); +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/index.tsx b/x-pack/plugins/data_usage/server/routes/internal/index.tsx new file mode 100644 index 0000000000000..d623dd8dd0ef7 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/index.tsx @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { registerUsageMetricsRoute } from './usage_metrics'; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts new file mode 100644 index 0000000000000..439b0d4561f12 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + DataUsageContext, + DataUsageRouter, + UsageMetricsRequestSchema, + UsageMetricsResponseSchema, +} from '../../types'; + +import { getUsageMetricsHandler } from './usage_metrics_handler'; + +export const registerUsageMetricsRoute = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + if (dataUsageContext.serverConfig.enabled) { + router.versioned + .get({ + access: 'internal', + path: '/internal/api/data_usage/metrics', + }) + .addVersion( + { + version: '1', + validate: { + request: UsageMetricsRequestSchema, + response: { + 200: UsageMetricsResponseSchema, + }, + }, + }, + getUsageMetricsHandler(dataUsageContext) + ); + } +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts new file mode 100644 index 0000000000000..6369b13aadcc4 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandler } from '@kbn/core/server'; +import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; +import { + DataUsageContext, + DataUsageRequestHandlerContext, + UsageMetricsRequestSchemaQueryParams, +} from '../../types'; + +import { errorHandler } from '../error_handler'; + +export const getUsageMetricsHandler = ( + dataUsageContext: DataUsageContext +): RequestHandler< + never, + UsageMetricsRequestSchemaQueryParams, + unknown, + DataUsageRequestHandlerContext +> => { + const logger = dataUsageContext.logFactory.get('usageMetricsRoute'); + + return async (context, request, response) => { + try { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + + // @ts-ignore + const { from, to, metricTypes, dataStreams: dsNames, size } = request.query; + logger.debug(`Retrieving usage metrics`); + + const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse = + await esClient.indices.getDataStream({ + name: '*', + expand_wildcards: 'all', + }); + + const hasDataStreams = dataStreamsResponse.length > 0; + let userDsNames: string[] = []; + + if (dsNames?.length) { + userDsNames = typeof dsNames === 'string' ? [dsNames] : dsNames; + } else if (!userDsNames.length && hasDataStreams) { + userDsNames = dataStreamsResponse.map((ds) => ds.name); + } + + // If no data streams are found, return an empty response + if (!userDsNames.length) { + return response.ok({ + body: { + charts: [], + }, + }); + } + // TODO: fetch data from autoOps using userDsNames + + // mock data + const charts = [ + { + key: 'ingest_rate', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 13756849 }, + { x: 1726862130000, y: 14657904 }, + { x: 1726865730000, y: 12798561 }, + { x: 1726869330000, y: 13578213 }, + { x: 1726872930000, y: 14123495 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + ], + }, + { + streamName: 'data_stream_3', + data: [{ x: 1726858530000, y: 12576413 }], + }, + ], + }, + { + key: 'storage_retained', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + ], + }, + ], + }, + ]; + return response.ok({ + body: { + charts, + }, + }); + } catch (error) { + return errorHandler(logger, response, error); + } + }; +}; diff --git a/x-pack/plugins/data_usage/server/types.ts b/x-pack/plugins/data_usage/server/types.ts deleted file mode 100644 index fa0d561701a9d..0000000000000 --- a/x-pack/plugins/data_usage/server/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; - -export interface DataUsageSetupDependencies { - features: FeaturesPluginSetup; -} - -/* eslint-disable @typescript-eslint/no-empty-interface*/ -export interface DataUsageStartDependencies {} - -export interface DataUsageServerSetup {} - -export interface DataUsageServerStart {} diff --git a/x-pack/plugins/data_usage/server/types/index.ts b/x-pack/plugins/data_usage/server/types/index.ts new file mode 100644 index 0000000000000..96577c72a8116 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './rest_types'; +export * from './types'; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/index.ts b/x-pack/plugins/data_usage/server/types/rest_types/index.ts new file mode 100644 index 0000000000000..2a6dc5006d53e --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './usage_metrics'; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts new file mode 100644 index 0000000000000..de46215a70d85 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UsageMetricsRequestSchema } from './usage_metrics'; + +describe('usage_metrics schemas', () => { + it('should accept valid request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + }) + ).not.toThrow(); + }); + + it('should accept a single `metricTypes` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'ingest_rate', + }) + ).not.toThrow(); + }); + + it('should accept multiple `metricTypes` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['ingest_rate', 'storage_retained', 'index_rate'], + }) + ).not.toThrow(); + }); + + it('should accept a single string as `dataStreams` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'storage_retained', + dataStreams: 'data_stream_1', + }) + ).not.toThrow(); + }); + + it('should accept valid `size`', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 100, + }) + ).not.toThrow(); + }); + + it('should accept `dataStream` list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 3, + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], + }) + ).not.toThrow(); + }); + + it('should error if `dataStream` list is empty', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 3, + dataStreams: [], + }) + ).toThrowError('expected value of type [string] but got [Array]'); + }); + + it('should error if `dataStream` is given an empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 1, + dataStreams: ' ', + }) + ).toThrow('[dataStreams] must have at least one value'); + }); + + it('should error if `dataStream` is given an empty item in the list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 3, + dataStreams: ['ds_1', ' '], + }) + ).toThrow('[dataStreams] list can not contain empty values'); + }); + + it('should error if `metricTypes` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ' ', + }) + ).toThrow(); + }); + + it('should error if `metricTypes` is empty item', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: [' ', 'storage_retained'], + }) + ).toThrow('[metricTypes] list can not contain empty values'); + }); + + it('should error if `metricTypes` is not a valid value', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'foo', + }) + ).toThrow( + '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' + ); + }); + + it('should error if `metricTypes` is not a valid list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow( + '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' + ); + }); + + it('should error if `from` is not valid', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: 'bar', + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + }) + ).toThrow('[from]: Invalid date'); + }); + + it('should error if `to` is not valid', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: 'foo', + metricTypes: ['storage_retained'], + }) + ).toThrow('[to]: Invalid date'); + }); +}); diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts new file mode 100644 index 0000000000000..5fe81567d94dc --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; + +const METRIC_TYPE_VALUES = [ + 'storage_retained', + 'ingest_rate', + 'search_vcu', + 'ingest_vcu', + 'ml_vcu', + 'index_latency', + 'index_rate', + 'search_latency', + 'search_rate', +] as const; + +// @ts-ignore +const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); + +const metricTypesSchema = { + // @ts-expect-error TS2769: No overload matches this call + schema: schema.oneOf(METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType))), + options: { minSize: 1, maxSize: METRIC_TYPE_VALUES.length }, +}; + +export const UsageMetricsRequestSchema = { + query: schema.object({ + from: schema.string({ + validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), + }), + to: schema.string({ + validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), + }), + size: schema.maybe(schema.number()), // should be same as dataStreams.length + metricTypes: schema.oneOf([ + schema.arrayOf(schema.string(), { + ...metricTypesSchema.options, + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[metricTypes] list can not contain empty values'; + } else if (values.map((v) => v.trim()).some((v) => !isValidMetricType(v))) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, + }), + schema.string({ + validate: (v) => { + if (!v.trim().length) { + return '[metricTypes] must have at least one value'; + } else if (!isValidMetricType(v)) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, + }), + ]), + dataStreams: schema.maybe( + schema.oneOf([ + schema.arrayOf(schema.string(), { + minSize: 1, + maxSize: 50, // TBD + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[dataStreams] list can not contain empty values'; + } + }, + }), + schema.string({ + validate: (v) => + v.trim().length ? undefined : '[dataStreams] must have at least one value', + }), + ]) + ), + }), +}; + +export type UsageMetricsRequestSchemaQueryParams = TypeOf; + +export const UsageMetricsResponseSchema = { + body: () => + schema.object({ + charts: schema.arrayOf( + schema.object({ + key: metricTypesSchema.schema, + series: schema.arrayOf( + schema.object({ + streamName: schema.string(), + data: schema.arrayOf( + schema.object({ + x: schema.number(), + y: schema.number(), + }) + ), + }) + ), + }), + { maxSize: 2 } + ), + }), +}; diff --git a/x-pack/plugins/data_usage/server/types/types.ts b/x-pack/plugins/data_usage/server/types/types.ts new file mode 100644 index 0000000000000..c90beb184f020 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + CoreRequestHandlerContext, + CustomRequestHandlerContext, + IRouter, + LoggerFactory, +} from '@kbn/core/server'; +import { DeepReadonly } from 'utility-types'; +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { DataUsageConfigType } from '../config'; + +export interface DataUsageSetupDependencies { + features: FeaturesPluginSetup; +} + +/* eslint-disable @typescript-eslint/no-empty-interface*/ +export interface DataUsageStartDependencies {} + +export interface DataUsageServerSetup {} + +export interface DataUsageServerStart {} + +interface DataUsageApiRequestHandlerContext { + core: CoreRequestHandlerContext; +} + +export type DataUsageRequestHandlerContext = CustomRequestHandlerContext<{ + dataUsage: DataUsageApiRequestHandlerContext; +}>; + +export type DataUsageRouter = IRouter; + +export interface DataUsageContext { + logFactory: LoggerFactory; + serverConfig: DeepReadonly; +} diff --git a/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts b/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts new file mode 100644 index 0000000000000..a7f00a0e82a3d --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class CustomHttpRequestError extends Error { + constructor( + message: string, + public readonly statusCode: number = 500, + public readonly meta?: unknown + ) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } + } +} diff --git a/x-pack/plugins/data_usage/server/utils/index.ts b/x-pack/plugins/data_usage/server/utils/index.ts new file mode 100644 index 0000000000000..78b123f7c2bdf --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { isValidateTimeRange } from './validate_time_range'; +export { CustomHttpRequestError } from './custom_http_request_error'; diff --git a/x-pack/plugins/data_usage/server/utils/validate_time_range.ts b/x-pack/plugins/data_usage/server/utils/validate_time_range.ts new file mode 100644 index 0000000000000..8a311c991e9fa --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/validate_time_range.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export const isValidateTimeRange = (fromTimestamp: number, endTimestamp: number) => { + if ( + fromTimestamp < endTimestamp && + fromTimestamp > 0 && + endTimestamp > 0 && + fromTimestamp !== endTimestamp + ) { + return true; + } + return false; +}; From 2a4de1731d63b93ea2c24a3b1e6d76cb5c7755fe Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Mon, 30 Sep 2024 15:41:51 +0200 Subject: [PATCH 12/53] data streams internal API --- .../data_usage/server/routes/index.tsx | 3 +- .../server/routes/internal/data_streams.ts | 35 +++++++++++++++ .../routes/internal/data_streams_handler.ts | 43 +++++++++++++++++++ .../server/routes/internal/index.tsx | 2 + .../server/types/rest_types/data_streams.ts | 18 ++++++++ .../server/types/rest_types/index.ts | 1 + 6 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/data_usage/server/routes/internal/data_streams.ts create mode 100644 x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts diff --git a/x-pack/plugins/data_usage/server/routes/index.tsx b/x-pack/plugins/data_usage/server/routes/index.tsx index b2d10b0ab53bc..b6b80c38864f3 100644 --- a/x-pack/plugins/data_usage/server/routes/index.tsx +++ b/x-pack/plugins/data_usage/server/routes/index.tsx @@ -6,11 +6,12 @@ */ import { DataUsageContext, DataUsageRouter } from '../types'; -import { registerUsageMetricsRoute } from './internal'; +import { registerDataStreamsRoute, registerUsageMetricsRoute } from './internal'; export const registerDataUsageRoutes = ( router: DataUsageRouter, dataUsageContext: DataUsageContext ) => { registerUsageMetricsRoute(router, dataUsageContext); + registerDataStreamsRoute(router, dataUsageContext); }; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts new file mode 100644 index 0000000000000..f3a401853f0ab --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataUsageContext, DataUsageRouter, DataStreamsResponseSchema } from '../../types'; + +import { getDataStreamsHandler } from './data_streams_handler'; + +export const registerDataStreamsRoute = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + if (dataUsageContext.serverConfig.enabled) { + router.versioned + .get({ + access: 'internal', + path: '/internal/api/data_usage/data_streams', + }) + .addVersion( + { + version: '1', + validate: { + request: {}, + response: { + 200: DataStreamsResponseSchema, + }, + }, + }, + getDataStreamsHandler(dataUsageContext) + ); + } +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts new file mode 100644 index 0000000000000..686edd0c4f4b7 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandler } from '@kbn/core/server'; +import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; + +import { errorHandler } from '../error_handler'; + +export const getDataStreamsHandler = ( + dataUsageContext: DataUsageContext +): RequestHandler => { + const logger = dataUsageContext.logFactory.get('dataStreamsRoute'); + + return async (context, _, response) => { + logger.debug(`Retrieving user data streams`); + + try { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + + const { data_streams: dataStreamsResponse } = await esClient.indices.dataStreamsStats({ + name: '*', + expand_wildcards: 'all', + }); + + const sorted = dataStreamsResponse + .sort((a, b) => b.store_size_bytes - a.store_size_bytes) + .map((dataStream) => ({ + name: dataStream.data_stream, + storageSizeBytes: dataStream.store_size_bytes, + })); + return response.ok({ + body: sorted, + }); + } catch (error) { + return errorHandler(logger, response, error); + } + }; +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/index.tsx b/x-pack/plugins/data_usage/server/routes/internal/index.tsx index d623dd8dd0ef7..e8d874bb7e6af 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/index.tsx +++ b/x-pack/plugins/data_usage/server/routes/internal/index.tsx @@ -4,4 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + export { registerUsageMetricsRoute } from './usage_metrics'; +export { registerDataStreamsRoute } from './data_streams'; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts b/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts new file mode 100644 index 0000000000000..b1c02bb40854d --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +export const DataStreamsResponseSchema = { + body: () => + schema.arrayOf( + schema.object({ + name: schema.string(), + storageSizeBytes: schema.number(), + }) + ), +}; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/index.ts b/x-pack/plugins/data_usage/server/types/rest_types/index.ts index 2a6dc5006d53e..64b5c640ebbb5 100644 --- a/x-pack/plugins/data_usage/server/types/rest_types/index.ts +++ b/x-pack/plugins/data_usage/server/types/rest_types/index.ts @@ -6,3 +6,4 @@ */ export * from './usage_metrics'; +export * from './data_streams'; From 36e0b02991aef2a32caa9a8120126e7007a88c6d Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 2 Oct 2024 09:25:58 +0200 Subject: [PATCH 13/53] public hook to fetch metrics data --- x-pack/plugins/data_usage/common/index.ts | 4 + .../rest_types/data_streams.ts | 0 .../types => common}/rest_types/index.ts | 0 .../rest_types/usage_metrics.test.ts | 0 .../rest_types/usage_metrics.ts | 5 +- .../public/hooks/use_get_usage_metrics.ts | 46 ++++ .../server/routes/internal/data_streams.ts | 6 +- .../server/routes/internal/usage_metrics.ts | 11 +- .../routes/internal/usage_metrics_handler.ts | 239 ++++++++++++------ .../plugins/data_usage/server/types/index.ts | 1 - 10 files changed, 227 insertions(+), 85 deletions(-) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/data_streams.ts (100%) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/index.ts (100%) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/usage_metrics.test.ts (100%) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/usage_metrics.ts (95%) create mode 100644 x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts diff --git a/x-pack/plugins/data_usage/common/index.ts b/x-pack/plugins/data_usage/common/index.ts index 4b6f899b58d37..eb0787f53f344 100644 --- a/x-pack/plugins/data_usage/common/index.ts +++ b/x-pack/plugins/data_usage/common/index.ts @@ -11,3 +11,7 @@ export const PLUGIN_ID = 'data_usage'; export const PLUGIN_NAME = i18n.translate('xpack.dataUsage.name', { defaultMessage: 'Data Usage', }); + +export const DATA_USAGE_API_ROUTE_PREFIX = '/api/data_usage/'; +export const DATA_USAGE_METRICS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}metrics`; +export const DATA_USAGE_DATA_STREAMS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}data_streams`; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts b/x-pack/plugins/data_usage/common/rest_types/data_streams.ts similarity index 100% rename from x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts rename to x-pack/plugins/data_usage/common/rest_types/data_streams.ts diff --git a/x-pack/plugins/data_usage/server/types/rest_types/index.ts b/x-pack/plugins/data_usage/common/rest_types/index.ts similarity index 100% rename from x-pack/plugins/data_usage/server/types/rest_types/index.ts rename to x-pack/plugins/data_usage/common/rest_types/index.ts diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts similarity index 100% rename from x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts rename to x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts similarity index 95% rename from x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts rename to x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 5fe81567d94dc..61dbf85e3516c 100644 --- a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -19,6 +19,8 @@ const METRIC_TYPE_VALUES = [ 'search_rate', ] as const; +export type MetricTypes = (typeof METRIC_TYPE_VALUES)[number]; + // @ts-ignore const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); @@ -62,7 +64,6 @@ export const UsageMetricsRequestSchema = { schema.oneOf([ schema.arrayOf(schema.string(), { minSize: 1, - maxSize: 50, // TBD validate: (values) => { if (values.map((v) => v.trim()).some((v) => !v.length)) { return '[dataStreams] list can not contain empty values'; @@ -102,3 +103,5 @@ export const UsageMetricsResponseSchema = { ), }), }; + +export type UsageMetricsResponseSchemaBody = TypeOf; diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts new file mode 100644 index 0000000000000..046579e6ac004 --- /dev/null +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; +import type { IHttpFetchError } from '@kbn/core-http-browser'; +import { + UsageMetricsRequestSchemaQueryParams, + UsageMetricsResponseSchemaBody, +} from '../../common/rest_types'; +import { DATA_USAGE_METRICS_API_ROUTE } from '../../common'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; + +interface ErrorType { + statusCode: number; + message: string; +} + +export const useGetDataUsageMetrics = ( + query: UsageMetricsRequestSchemaQueryParams, + options: UseQueryOptions> = {} +): UseQueryResult> => { + const http = useKibanaContextForPlugin().services.http; + + return useQuery>({ + queryKey: ['get-data-usage-metrics', query], + ...options, + keepPreviousData: true, + queryFn: async () => { + return http.get(DATA_USAGE_METRICS_API_ROUTE, { + version: '2023-10-31', + query: { + from: query.from, + to: query.to, + metricTypes: query.metricTypes, + size: query.size, + dataStreams: query.dataStreams, + }, + }); + }, + }); +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts index f3a401853f0ab..0d71d93b55849 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts @@ -5,7 +5,9 @@ * 2.0. */ -import { DataUsageContext, DataUsageRouter, DataStreamsResponseSchema } from '../../types'; +import { DataStreamsResponseSchema } from '../../../common/rest_types'; +import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../../common'; +import { DataUsageContext, DataUsageRouter } from '../../types'; import { getDataStreamsHandler } from './data_streams_handler'; @@ -17,7 +19,7 @@ export const registerDataStreamsRoute = ( router.versioned .get({ access: 'internal', - path: '/internal/api/data_usage/data_streams', + path: DATA_USAGE_DATA_STREAMS_API_ROUTE, }) .addVersion( { diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts index 439b0d4561f12..5bf3008ef668a 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -5,12 +5,9 @@ * 2.0. */ -import { - DataUsageContext, - DataUsageRouter, - UsageMetricsRequestSchema, - UsageMetricsResponseSchema, -} from '../../types'; +import { UsageMetricsRequestSchema, UsageMetricsResponseSchema } from '../../../common/rest_types'; +import { DATA_USAGE_METRICS_API_ROUTE } from '../../../common'; +import { DataUsageContext, DataUsageRouter } from '../../types'; import { getUsageMetricsHandler } from './usage_metrics_handler'; @@ -22,7 +19,7 @@ export const registerUsageMetricsRoute = ( router.versioned .get({ access: 'internal', - path: '/internal/api/data_usage/metrics', + path: DATA_USAGE_METRICS_API_ROUTE, }) .addVersion( { diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 6369b13aadcc4..6ae9b0ffbe82d 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -7,14 +7,14 @@ import { RequestHandler } from '@kbn/core/server'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; -import { - DataUsageContext, - DataUsageRequestHandlerContext, - UsageMetricsRequestSchemaQueryParams, -} from '../../types'; +import { MetricTypes, UsageMetricsRequestSchemaQueryParams } from '../../../common/rest_types'; +import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; import { errorHandler } from '../error_handler'; +const formatStringParams = (value: T | T[]): T[] | MetricTypes[] => + typeof value === 'string' ? [value] : value; + export const getUsageMetricsHandler = ( dataUsageContext: DataUsageContext ): RequestHandler< @@ -57,76 +57,15 @@ export const getUsageMetricsHandler = ( }, }); } - // TODO: fetch data from autoOps using userDsNames - // mock data - const charts = [ - { - key: 'ingest_rate', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 13756849 }, - { x: 1726862130000, y: 14657904 }, - { x: 1726865730000, y: 12798561 }, - { x: 1726869330000, y: 13578213 }, - { x: 1726872930000, y: 14123495 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - ], - }, - { - streamName: 'data_stream_3', - data: [{ x: 1726858530000, y: 12576413 }], - }, - ], - }, - { - key: 'storage_retained', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - ], - }, - { - streamName: 'data_stream_3', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - ], - }, - ], - }, - ]; + const charts = await fetchMetricsFromAutoOps({ + from, + to, + size, + metricTypes: formatStringParams(metricTypes) as MetricTypes[], + dataStreams: formatStringParams(userDsNames), + }); + return response.ok({ body: { charts, @@ -137,3 +76,155 @@ export const getUsageMetricsHandler = ( } }; }; + +const fetchMetricsFromAutoOps = async ({ + from, + to, + size, + metricTypes, + dataStreams, +}: { + from: string; + to: string; + size?: number; + metricTypes: MetricTypes[]; + dataStreams: string[]; +}) => { + // TODO: fetch data from autoOps using userDsNames + + const charts = [ + { + key: 'ingest_rate', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 13756849 }, + { x: 1726862130000, y: 14657904 }, + { x: 1726865730000, y: 12798561 }, + { x: 1726869330000, y: 13578213 }, + { x: 1726872930000, y: 14123495 }, + { x: 1726876530000, y: 13876548 }, + { x: 1726880130000, y: 12894561 }, + { x: 1726883730000, y: 14478953 }, + { x: 1726887330000, y: 14678905 }, + { x: 1726890930000, y: 13976547 }, + { x: 1726894530000, y: 14568945 }, + { x: 1726898130000, y: 13789561 }, + { x: 1726901730000, y: 14478905 }, + { x: 1726905330000, y: 13956423 }, + { x: 1726908930000, y: 14598234 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + ], + }, + { + key: 'storage_retained', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + // Repeat similar structure for more data streams... + ], + }, + ]; + + return charts; +}; diff --git a/x-pack/plugins/data_usage/server/types/index.ts b/x-pack/plugins/data_usage/server/types/index.ts index 96577c72a8116..6cc0ccaa93a6d 100644 --- a/x-pack/plugins/data_usage/server/types/index.ts +++ b/x-pack/plugins/data_usage/server/types/index.ts @@ -5,5 +5,4 @@ * 2.0. */ -export * from './rest_types'; export * from './types'; From 8aee1e868d67036ec33a06e413a2b23714667b5c Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 2 Oct 2024 18:22:32 +0200 Subject: [PATCH 14/53] plug metrics API with charts UI --- .../data_usage/common/query_client.tsx | 57 +++++++ .../public/app/components/charts.tsx | 6 +- .../data_usage/public/app/data_usage.tsx | 146 ++---------------- x-pack/plugins/data_usage/public/app/types.ts | 5 +- .../plugins/data_usage/public/application.tsx | 5 +- .../public/hooks/use_get_usage_metrics.ts | 2 +- 6 files changed, 77 insertions(+), 144 deletions(-) create mode 100644 x-pack/plugins/data_usage/common/query_client.tsx diff --git a/x-pack/plugins/data_usage/common/query_client.tsx b/x-pack/plugins/data_usage/common/query_client.tsx new file mode 100644 index 0000000000000..8a64ed7a51349 --- /dev/null +++ b/x-pack/plugins/data_usage/common/query_client.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PropsWithChildren } from 'react'; +import React, { memo, useMemo } from 'react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; + +type QueryClientOptionsProp = ConstructorParameters[0]; + +/** + * Default Query Client for Data Usage. + */ +export class DataUsageQueryClient extends QueryClient { + constructor(options: QueryClientOptionsProp = {}) { + const optionsWithDefaults: QueryClientOptionsProp = { + ...options, + defaultOptions: { + ...(options.defaultOptions ?? {}), + queries: { + refetchIntervalInBackground: false, + refetchOnWindowFocus: false, + refetchOnMount: true, + keepPreviousData: true, + ...(options?.defaultOptions?.queries ?? {}), + }, + }, + }; + super(optionsWithDefaults); + } +} + +/** + * The default Data Usage Query Client. Can be imported and used from outside of React hooks + * and still benefit from ReactQuery features (like caching, etc) + * + * @see https://tanstack.com/query/v4/docs/reference/QueryClient + */ +export const dataUsageQueryClient = new DataUsageQueryClient(); + +export type ReactQueryClientProviderProps = PropsWithChildren<{ + queryClient?: DataUsageQueryClient; +}>; + +export const DataUsageReactQueryClientProvider = memo( + ({ queryClient, children }) => { + const client = useMemo(() => { + return queryClient || dataUsageQueryClient; + }, [queryClient]); + return {children}; + } +); + +DataUsageReactQueryClientProvider.displayName = 'DataUsageReactQueryClientProvider'; diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 08f51f60f887c..41b63d7623e88 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -40,11 +40,11 @@ const formatBytes = (bytes: number) => { return numeral(bytes).format('0.0 b'); }; -export const chartKeyToTitleMap: Record = { - ingestedMax: i18n.translate('xpack.dataUsage.charts.ingestedMax', { +export const chartKeyToTitleMap: Record = { + ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { defaultMessage: 'Data Ingested', }), - retainedMax: i18n.translate('xpack.dataUsage.charts.retainedMax', { + storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { defaultMessage: 'Data Retained in Storage', }), }; diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 476caaa8930a7..4c42f1b6141b2 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -18,153 +18,25 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { Charts } from './components/charts'; import { DatePicker } from './components/date_picker'; -import { MetricsResponse } from './types'; import { useBreadcrumbs } from '../utils/use_breadcrumbs'; import { useKibanaContextForPlugin } from '../utils/use_kibana'; import { PLUGIN_NAME } from '../../common'; import { DatePickerProvider } from './hooks/use_date_picker'; - -const response: MetricsResponse = { - charts: [ - { - key: 'ingestedMax', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 13756849 }, - { x: 1726862130000, y: 14657904 }, - { x: 1726865730000, y: 12798561 }, - { x: 1726869330000, y: 13578213 }, - { x: 1726872930000, y: 14123495 }, - { x: 1726876530000, y: 13876548 }, - { x: 1726880130000, y: 12894561 }, - { x: 1726883730000, y: 14478953 }, - { x: 1726887330000, y: 14678905 }, - { x: 1726890930000, y: 13976547 }, - { x: 1726894530000, y: 14568945 }, - { x: 1726898130000, y: 13789561 }, - { x: 1726901730000, y: 14478905 }, - { x: 1726905330000, y: 13956423 }, - { x: 1726908930000, y: 14598234 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - { x: 1726869330000, y: 14048532 }, - { x: 1726872930000, y: 14237495 }, - { x: 1726876530000, y: 13745689 }, - { x: 1726880130000, y: 13974562 }, - { x: 1726883730000, y: 14234653 }, - { x: 1726887330000, y: 14323479 }, - { x: 1726890930000, y: 14023945 }, - { x: 1726894530000, y: 14189673 }, - { x: 1726898130000, y: 14247895 }, - { x: 1726901730000, y: 14098324 }, - { x: 1726905330000, y: 14478905 }, - { x: 1726908930000, y: 14323894 }, - ], - }, - { - streamName: 'data_stream_3', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, - ], - }, - ], - }, - { - key: 'retainedMax', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - { x: 1726869330000, y: 14048532 }, - { x: 1726872930000, y: 14237495 }, - { x: 1726876530000, y: 13745689 }, - { x: 1726880130000, y: 13974562 }, - { x: 1726883730000, y: 14234653 }, - { x: 1726887330000, y: 14323479 }, - { x: 1726890930000, y: 14023945 }, - { x: 1726894530000, y: 14189673 }, - { x: 1726898130000, y: 14247895 }, - { x: 1726901730000, y: 14098324 }, - { x: 1726905330000, y: 14478905 }, - { x: 1726908930000, y: 14323894 }, - ], - }, - { - streamName: 'data_stream_3', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, - ], - }, - // Repeat similar structure for more data streams... - ], - }, - ], -}; +import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; export const DataUsage = () => { const { services: { chrome, appParams }, } = useKibanaContextForPlugin(); + const { data, isFetching, isError } = useGetDataUsageMetrics({ + metricTypes: ['storage_retained', 'ingest_rate'], + to: 1726908930000, + from: 1726858530000, + }); + + const isLoading = isFetching || !data; + useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); return ( diff --git a/x-pack/plugins/data_usage/public/app/types.ts b/x-pack/plugins/data_usage/public/app/types.ts index 279a7e0e863ef..74e2d285f232b 100644 --- a/x-pack/plugins/data_usage/public/app/types.ts +++ b/x-pack/plugins/data_usage/public/app/types.ts @@ -4,7 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { MetricKey } from '../../common/types'; + +import { MetricTypes } from '../../common/rest_types'; export interface DataPoint { x: number; @@ -16,7 +17,7 @@ export interface Series { data: DataPoint[]; } export interface Chart { - key: MetricKey; + key: MetricTypes; series: Series[]; } diff --git a/x-pack/plugins/data_usage/public/application.tsx b/x-pack/plugins/data_usage/public/application.tsx index a160a5742d36d..054aae397e5e1 100644 --- a/x-pack/plugins/data_usage/public/application.tsx +++ b/x-pack/plugins/data_usage/public/application.tsx @@ -17,6 +17,7 @@ import { useKibanaContextForPluginProvider } from './utils/use_kibana'; import { DataUsageStartDependencies, DataUsagePublicStart } from './types'; import { PLUGIN_ID } from '../common'; import { DataUsage } from './app/data_usage'; +import { DataUsageReactQueryClientProvider } from '../common/query_client'; export const renderApp = ( core: CoreStart, @@ -77,7 +78,9 @@ const App = ({ core, plugins, pluginStart, params }: AppProps) => { return ( - + + + ); diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts index 046579e6ac004..91cee020f243e 100644 --- a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -32,7 +32,7 @@ export const useGetDataUsageMetrics = ( keepPreviousData: true, queryFn: async () => { return http.get(DATA_USAGE_METRICS_API_ROUTE, { - version: '2023-10-31', + version: '1', query: { from: query.from, to: query.to, From fdfa96b5dbf51a98b012606721eaf6e2a001c3a3 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Fri, 4 Oct 2024 09:17:14 +0200 Subject: [PATCH 15/53] update date picker to work with API and update URL --- .../common/rest_types/usage_metrics.test.ts | 22 --- .../common/rest_types/usage_metrics.ts | 14 +- x-pack/plugins/data_usage/kibana.jsonc | 20 ++- .../public/app/components/charts.tsx | 6 +- .../public/app/components/date_picker.tsx | 118 +++++++++----- .../data_usage/public/app/data_usage.tsx | 71 +++++++-- .../app/hooks/use_charts_url_params.tsx | 146 ++++++++++++++++++ .../public/app/hooks/use_date_picker.tsx | 111 ++++++++++--- .../data_usage/public/hooks/use_url_params.ts | 30 ++++ x-pack/plugins/data_usage/tsconfig.json | 1 + 10 files changed, 430 insertions(+), 109 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx create mode 100644 x-pack/plugins/data_usage/public/hooks/use_url_params.ts diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts index de46215a70d85..1e04c84c113d1 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts @@ -11,8 +11,6 @@ describe('usage_metrics schemas', () => { it('should accept valid request query', () => { expect(() => UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: new Date().toISOString(), metricTypes: ['storage_retained'], }) ).not.toThrow(); @@ -151,24 +149,4 @@ describe('usage_metrics schemas', () => { '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' ); }); - - it('should error if `from` is not valid', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: 'bar', - to: new Date().toISOString(), - metricTypes: ['storage_retained'], - }) - ).toThrow('[from]: Invalid date'); - }); - - it('should error if `to` is not valid', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: 'foo', - metricTypes: ['storage_retained'], - }) - ).toThrow('[to]: Invalid date'); - }); }); diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 61dbf85e3516c..0c7971686ea43 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -21,9 +21,15 @@ const METRIC_TYPE_VALUES = [ export type MetricTypes = (typeof METRIC_TYPE_VALUES)[number]; +// type guard for MetricTypes +export const isMetricType = (type: string): type is MetricTypes => + METRIC_TYPE_VALUES.includes(type as MetricTypes); + // @ts-ignore const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); +const DateSchema = schema.maybe(schema.string()); + const metricTypesSchema = { // @ts-expect-error TS2769: No overload matches this call schema: schema.oneOf(METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType))), @@ -32,12 +38,8 @@ const metricTypesSchema = { export const UsageMetricsRequestSchema = { query: schema.object({ - from: schema.string({ - validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), - }), - to: schema.string({ - validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), - }), + from: DateSchema, + to: DateSchema, size: schema.maybe(schema.number()), // should be same as dataStreams.length metricTypes: schema.oneOf([ schema.arrayOf(schema.string(), { diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 9b0f2d193925e..ffd8833351267 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -1,16 +1,28 @@ { "type": "plugin", "id": "@kbn/data-usage-plugin", - "owner": ["@elastic/obs-ai-assistant", "@elastic/security-solution"], + "owner": [ + "@elastic/obs-ai-assistant", + "@elastic/security-solution" + ], "plugin": { "id": "dataUsage", "server": true, "browser": true, - "configPath": ["xpack", "dataUsage"], - "requiredPlugins": ["home", "management", "features", "share"], + "configPath": [ + "xpack", + "dataUsage" + ], + "requiredPlugins": [ + "home", + "management", + "features", + "share" + ], "optionalPlugins": [], "requiredBundles": [ "kibanaReact", - ], + "data" + ] } } diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 41b63d7623e88..48e04c7906256 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -31,8 +31,8 @@ import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { MetricsResponse } from '../types'; -import { MetricKey } from '../../../common/types'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +import { MetricTypes } from '../../../common/rest_types'; interface ChartsProps { data: MetricsResponse; } @@ -40,7 +40,9 @@ const formatBytes = (bytes: number) => { return numeral(bytes).format('0.0 b'); }; -export const chartKeyToTitleMap: Record = { +// TODO: Remove this when we have a title for each metric type +type ChartKey = Extract; +export const chartKeyToTitleMap: Record = { ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { defaultMessage: 'Data Ingested', }), diff --git a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx index f95af6d6bfc43..ca29acf8c96a6 100644 --- a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx +++ b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx @@ -5,48 +5,84 @@ * 2.0. */ -import React from 'react'; -import { EuiSuperDatePicker } from '@elastic/eui'; -import { useDatePickerContext } from '../hooks/use_date_picker'; +import React, { memo, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiSuperDatePicker, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/react'; +import type { IUnifiedSearchPluginServices } from '@kbn/unified-search-plugin/public'; +import type { EuiSuperDatePickerRecentRange } from '@elastic/eui'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import type { + DurationRange, + OnRefreshChangeProps, +} from '@elastic/eui/src/components/date_picker/types'; +import { UI_SETTINGS } from '@kbn/data-plugin/common'; -export const DatePicker: React.FC = () => { - const { - startDate, - endDate, - isAutoRefreshActive, - refreshInterval, - setStartDate, - setEndDate, - setIsAutoRefreshActive, - setRefreshInterval, - } = useDatePickerContext(); - - const onTimeChange = ({ start, end }: { start: string; end: string }) => { - setStartDate(start); - setEndDate(end); - // Trigger API call or data refresh here +export interface DateRangePickerValues { + autoRefreshOptions: { + enabled: boolean; + duration: number; }; + startDate: string; + endDate: string; + recentlyUsedDateRanges: EuiSuperDatePickerRecentRange[]; +} - const onRefreshChange = ({ - isPaused, - refreshInterval: onRefreshRefreshInterval, - }: { - isPaused: boolean; - refreshInterval: number; - }) => { - setIsAutoRefreshActive(!isPaused); - setRefreshInterval(onRefreshRefreshInterval); - }; +interface UsageMetricsDateRangePickerProps { + dateRangePickerState: DateRangePickerValues; + isDataLoading: boolean; + onRefresh: () => void; + onRefreshChange: (evt: OnRefreshChangeProps) => void; + onTimeChange: ({ start, end }: DurationRange) => void; +} + +export const UsageMetricsDateRangePicker = memo( + ({ dateRangePickerState, isDataLoading, onRefresh, onRefreshChange, onTimeChange }) => { + const { euiTheme } = useEuiTheme(); + const kibana = useKibana(); + const { uiSettings } = kibana.services; + const [commonlyUsedRanges] = useState(() => { + return ( + uiSettings + ?.get(UI_SETTINGS.TIMEPICKER_QUICK_RANGES) + ?.map(({ from, to, display }: { from: string; to: string; display: string }) => { + return { + start: from, + end: to, + label: display, + }; + }) ?? [] + ); + }); + + return ( +
+ + + + + +
+ ); + } +); - return ( - - ); -}; +UsageMetricsDateRangePicker.displayName = 'UsageMetricsDateRangePicker'; diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 4c42f1b6141b2..77104eeb1d5c3 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -5,42 +5,81 @@ * 2.0. */ -import React from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem, - EuiText, + EuiLoadingElastic, EuiPageSection, + EuiText, } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { UsageMetricsRequestSchemaQueryParams } from '../../common/rest_types'; import { Charts } from './components/charts'; -import { DatePicker } from './components/date_picker'; +import { UsageMetricsDateRangePicker } from './components/date_picker'; import { useBreadcrumbs } from '../utils/use_breadcrumbs'; import { useKibanaContextForPlugin } from '../utils/use_kibana'; import { PLUGIN_NAME } from '../../common'; -import { DatePickerProvider } from './hooks/use_date_picker'; import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; +import { useDateRangePicker } from './hooks/use_date_picker'; +import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params'; export const DataUsage = () => { const { services: { chrome, appParams }, } = useKibanaContextForPlugin(); - const { data, isFetching, isError } = useGetDataUsageMetrics({ + const { metricTypes: metricTypesFromUrl, dataStreams: dataStreamsFromUrl } = + useDataUsageMetricsUrlParams(); + + const [queryParams, setQueryParams] = useState({ metricTypes: ['storage_retained', 'ingest_rate'], - to: 1726908930000, - from: 1726858530000, + dataStreams: [], }); - const isLoading = isFetching || !data; + useEffect(() => { + setQueryParams((prevState) => ({ + ...prevState, + metricTypes: metricTypesFromUrl?.length ? metricTypesFromUrl : prevState.metricTypes, + dataStreams: dataStreamsFromUrl?.length ? dataStreamsFromUrl : prevState.dataStreams, + })); + }, [metricTypesFromUrl, dataStreamsFromUrl]); + + const { dateRangePickerState, onRefreshChange, onTimeChange } = useDateRangePicker(); + + const { + error, + data, + isFetching, + isFetched, + refetch: refetchDataUsageMetrics, + } = useGetDataUsageMetrics( + { + ...queryParams, + from: dateRangePickerState.startDate, + to: dateRangePickerState.endDate, + }, + { + retry: false, + } + ); + + const onRefresh = useCallback(() => { + refetchDataUsageMetrics(); + }, [refetchDataUsageMetrics]); useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); + // TODO: show a toast? + if (!isFetching && error?.body) { + return
{error.body.message}
; + } + return ( - + <>

{i18n.translate('xpack.dataUsage.pageTitle', { @@ -60,12 +99,18 @@ export const DataUsage = () => { - + - + {isFetched && data ? : } - + ); }; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx new file mode 100644 index 0000000000000..9b31e10b9e18e --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useHistory, useLocation } from 'react-router-dom'; +import { MetricTypes, isMetricType } from '../../../common/rest_types'; +import { useUrlParams } from '../../hooks/use_url_params'; + +interface UrlParamsDataUsageMetricsFilters { + metricTypes: string; + dataStreams: string; + startDate: string; + endDate: string; +} + +interface DataUsageMetricsFiltersFromUrlParams { + metricTypes?: MetricTypes[]; + dataStreams?: string[]; + startDate?: string; + endDate?: string; + setUrlDataStreamsFilter: (dataStreams: UrlParamsDataUsageMetricsFilters['dataStreams']) => void; + setUrlDateRangeFilter: ({ startDate, endDate }: { startDate: string; endDate: string }) => void; + setUrlMetricTypesFilter: (metricTypes: UrlParamsDataUsageMetricsFilters['metricTypes']) => void; +} + +type FiltersFromUrl = Pick< + DataUsageMetricsFiltersFromUrlParams, + 'metricTypes' | 'dataStreams' | 'startDate' | 'endDate' +>; + +export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({ + autoRefreshOptions: { + enabled: false, + duration: 10000, + }, + startDate: 'now-24h/h', + endDate: 'now', + recentlyUsedDateRanges: [], +}); + +export const getDataUsageMetricsFiltersFromUrlParams = ( + urlParams: Partial +): FiltersFromUrl => { + const dataUsageMetricsFilters: FiltersFromUrl = { + metricTypes: [], + dataStreams: [], + startDate: DEFAULT_DATE_RANGE_OPTIONS.startDate, + endDate: DEFAULT_DATE_RANGE_OPTIONS.endDate, + }; + + const urlMetricTypes = urlParams.metricTypes + ? (String(urlParams.metricTypes).split(',') as MetricTypes[]).reduce( + (acc, curr) => { + if (isMetricType(curr)) { + acc.push(curr); + } + return acc.sort(); + }, + [] + ) + : []; + + const urlDataStreams = urlParams.dataStreams + ? String(urlParams.dataStreams).split(',').sort() + : []; + + dataUsageMetricsFilters.metricTypes = urlMetricTypes.length ? urlMetricTypes : undefined; + dataUsageMetricsFilters.dataStreams = urlDataStreams.length ? urlDataStreams : undefined; + dataUsageMetricsFilters.startDate = urlParams.startDate ? String(urlParams.startDate) : undefined; + dataUsageMetricsFilters.endDate = urlParams.endDate ? String(urlParams.endDate) : undefined; + + return dataUsageMetricsFilters; +}; + +export const useDataUsageMetricsUrlParams = (): DataUsageMetricsFiltersFromUrlParams => { + const location = useLocation(); + const history = useHistory(); + const { urlParams, toUrlParams } = useUrlParams(); + + const getUrlDataUsageMetricsFilters: FiltersFromUrl = useMemo( + () => getDataUsageMetricsFiltersFromUrlParams(urlParams), + [urlParams] + ); + const [dataUsageMetricsFilters, setDataUsageMetricsFilters] = useState( + getUrlDataUsageMetricsFilters + ); + + const setUrlMetricTypesFilter = useCallback( + (metricTypes: string) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + hosts: metricTypes.length ? metricTypes : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + const setUrlDataStreamsFilter = useCallback( + (dataStreams: string) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + dataStreams: dataStreams.length ? dataStreams : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + const setUrlDateRangeFilter = useCallback( + ({ startDate, endDate }: { startDate: string; endDate: string }) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + startDate: startDate.length ? startDate : undefined, + endDate: endDate.length ? endDate : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + useEffect(() => { + setDataUsageMetricsFilters((prevState) => { + return { + ...prevState, + ...getDataUsageMetricsFiltersFromUrlParams(urlParams), + }; + }); + }, [setDataUsageMetricsFilters, urlParams]); + + return { + ...dataUsageMetricsFilters, + setUrlDataStreamsFilter, + setUrlDateRangeFilter, + setUrlMetricTypesFilter, + }; +}; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx index cd8ac35c98839..b5407ae9e46d5 100644 --- a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx +++ b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx @@ -5,25 +5,94 @@ * 2.0. */ -import { useState } from 'react'; -import createContainer from 'constate'; - -const useDatePicker = () => { - const [startDate, setStartDate] = useState('now-24h'); - const [endDate, setEndDate] = useState('now'); - const [isAutoRefreshActive, setIsAutoRefreshActive] = useState(false); - const [refreshInterval, setRefreshInterval] = useState(0); - - return { - startDate, - setStartDate, - endDate, - setEndDate, - isAutoRefreshActive, - setIsAutoRefreshActive, - refreshInterval, - setRefreshInterval, - }; -}; +import { useCallback, useState } from 'react'; +import type { + DurationRange, + OnRefreshChangeProps, +} from '@elastic/eui/src/components/date_picker/types'; +import { useDataUsageMetricsUrlParams } from './use_charts_url_params'; +import { DateRangePickerValues } from '../components/date_picker'; + +export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({ + autoRefreshOptions: { + enabled: false, + duration: 10000, + }, + startDate: 'now-24h/h', + endDate: 'now', + recentlyUsedDateRanges: [], +}); + +export const useDateRangePicker = () => { + const { + setUrlDateRangeFilter, + startDate: startDateFromUrl, + endDate: endDateFromUrl, + } = useDataUsageMetricsUrlParams(); + const [dateRangePickerState, setDateRangePickerState] = useState({ + ...DEFAULT_DATE_RANGE_OPTIONS, + startDate: startDateFromUrl ?? DEFAULT_DATE_RANGE_OPTIONS.startDate, + endDate: endDateFromUrl ?? DEFAULT_DATE_RANGE_OPTIONS.endDate, + }); + + const updateUsageMetricsDateRanges = useCallback( + ({ start, end }: DurationRange) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + startDate: start, + endDate: end, + })); + }, + [setDateRangePickerState] + ); + + const updateUsageMetricsRecentlyUsedDateRanges = useCallback( + (recentlyUsedDateRanges: DateRangePickerValues['recentlyUsedDateRanges']) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + recentlyUsedDateRanges, + })); + }, + [setDateRangePickerState] + ); -export const [DatePickerProvider, useDatePickerContext] = createContainer(useDatePicker); + // handle refresh timer update + const onRefreshChange = useCallback( + (evt: OnRefreshChangeProps) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + autoRefreshOptions: { enabled: !evt.isPaused, duration: evt.refreshInterval }, + })); + }, + [setDateRangePickerState] + ); + + // handle manual time change on date picker + const onTimeChange = useCallback( + ({ start: newStart, end: newEnd }: DurationRange) => { + // update date ranges + updateUsageMetricsDateRanges({ start: newStart, end: newEnd }); + + // update recently used date ranges + const newRecentlyUsedDateRanges = [ + { start: newStart, end: newEnd }, + ...dateRangePickerState.recentlyUsedDateRanges + .filter( + (recentlyUsedRange: DurationRange) => + !(recentlyUsedRange.start === newStart && recentlyUsedRange.end === newEnd) + ) + .slice(0, 9), + ]; + updateUsageMetricsRecentlyUsedDateRanges(newRecentlyUsedDateRanges); + setUrlDateRangeFilter({ startDate: newStart, endDate: newEnd }); + }, + [ + dateRangePickerState.recentlyUsedDateRanges, + setUrlDateRangeFilter, + updateUsageMetricsDateRanges, + updateUsageMetricsRecentlyUsedDateRanges, + ] + ); + + return { dateRangePickerState, onRefreshChange, onTimeChange }; +}; diff --git a/x-pack/plugins/data_usage/public/hooks/use_url_params.ts b/x-pack/plugins/data_usage/public/hooks/use_url_params.ts new file mode 100644 index 0000000000000..865b71781df63 --- /dev/null +++ b/x-pack/plugins/data_usage/public/hooks/use_url_params.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import { parse, stringify } from 'query-string'; +import { useLocation } from 'react-router-dom'; + +/** + * Parses `search` params and returns an object with them along with a `toUrlParams` function + * that allows being able to retrieve a stringified version of an object (default is the + * `urlParams` that was parsed) for use in the url. + * Object will be recreated every time `search` changes. + */ +export function useUrlParams>(): { + urlParams: T; + toUrlParams: (params?: T) => string; +} { + const { search } = useLocation(); + return useMemo(() => { + const urlParams = parse(search) as unknown as T; + return { + urlParams, + toUrlParams: (params: T = urlParams) => stringify(params as unknown as object), + }; + }, [search]); +} diff --git a/x-pack/plugins/data_usage/tsconfig.json b/x-pack/plugins/data_usage/tsconfig.json index ebc023568cf88..9087062db0b06 100644 --- a/x-pack/plugins/data_usage/tsconfig.json +++ b/x-pack/plugins/data_usage/tsconfig.json @@ -13,6 +13,7 @@ "kbn_references": [ "@kbn/core", "@kbn/i18n", + "@kbn/data-plugin", "@kbn/kibana-react-plugin", "@kbn/management-plugin", "@kbn/react-kibana-context-render", From 63db939290fb8948a1a34a3023cb0cb97a60c444 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 12:05:22 -0400 Subject: [PATCH 16/53] add kibana index link and separate out dataset quality link component --- .../public/app/components/charts.tsx | 47 +++++++------------ .../app/components/dataset_quality_link.tsx | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 48e04c7906256..9d198a348dc80 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -29,10 +29,10 @@ import { } from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; -import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { MetricsResponse } from '../types'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; import { MetricTypes } from '../../../common/rest_types'; +import { DatasetQualityLink } from './dataset_quality_link'; interface ChartsProps { data: MetricsResponse; } @@ -50,13 +50,7 @@ export const chartKeyToTitleMap: Record = { defaultMessage: 'Data Retained in Storage', }), }; -function getDatasetFromDataStream(dataStreamName: string): string | null { - const parts = dataStreamName.split('-'); - if (parts.length !== 3) { - return null; - } - return parts[1]; -} + export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); const theme = useEuiTheme(); @@ -69,20 +63,16 @@ export const Charts: React.FC = ({ data }) => { }, } = useKibanaContextForPlugin(); const hasDataSetQualityFeature = capabilities.data_quality; - const onClickDataQuality = async ({ dataStreamName }: { dataStreamName: string }) => { - const dataQualityLocator = locators.get(DATA_QUALITY_LOCATOR_ID); - const locatorParams: DataQualityLocatorParams = { - filters: { - // TODO: get time range from our page state - timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, - }, - }; - const dataset = getDatasetFromDataStream(dataStreamName); - if (locatorParams?.filters && dataset) { - locatorParams.filters.query = dataset; - } + const hasIndexManagementFeature = capabilities.index_management; + + const onClickIndexManagement = async ({ dataStreamName }: { dataStreamName: string }) => { + // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 + const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); if (dataQualityLocator) { - await dataQualityLocator.navigate(locatorParams); + await dataQualityLocator.navigate({ + sectionId: 'data', + appId: `index_management/data_streams/${dataStreamName}`, + }); } }; @@ -136,21 +126,20 @@ export const Charts: React.FC = ({ data }) => { label="Copy data stream name" onClick={() => undefined} /> - undefined} - /> - {hasDataSetQualityFeature && ( + {hasIndexManagementFeature && ( - onClickDataQuality({ + onClickIndexManagement({ dataStreamName: label, }) } /> )} + {hasDataSetQualityFeature && ( + + )} diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx new file mode 100644 index 0000000000000..f095563a65a6e --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiListGroupItem } from '@elastic/eui'; + +import React from 'react'; +import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; + +function getDatasetFromDataStream(dataStreamName: string): string | null { + const parts = dataStreamName.split('-'); + if (parts.length !== 3) { + return null; + } + return parts[1]; +} + +export const DatasetQualityLink = React.memo(({ dataStreamName }: { dataStreamName: string }) => { + const { + services: { + share: { url }, + }, + } = useKibanaContextForPlugin(); + + const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); + const onClickDataQuality = async () => { + const locatorParams: DataQualityLocatorParams = { + filters: { + // TODO: get time range from our page state + timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + }, + }; + const dataset = getDatasetFromDataStream(dataStreamName); + if (locatorParams?.filters && dataset) { + locatorParams.filters.query = dataset; + } + if (locator) { + await locator.navigate(locatorParams); + } + }; + + return onClickDataQuality()} />; +}); From 38d338be8d222234c4b1918547ef4cd9e57a8282 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 12:43:59 -0400 Subject: [PATCH 17/53] app copy function and spacer --- .../plugins/data_usage/public/app/components/charts.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 9d198a348dc80..956cfec193636 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -13,6 +13,7 @@ import { EuiListGroupItem, EuiPanel, EuiPopover, + EuiSpacer, EuiTitle, useEuiTheme, } from '@elastic/eui'; @@ -122,10 +123,13 @@ export const Charts: React.FC = ({ data }) => { > undefined} + onClick={() => { + navigator.clipboard.writeText(label); + }} /> + + {hasIndexManagementFeature && ( Date: Fri, 4 Oct 2024 13:29:13 -0400 Subject: [PATCH 18/53] create LegendAction component --- .../public/app/components/charts.tsx | 100 ++++-------------- .../public/app/components/legend_action.tsx | 95 +++++++++++++++++ 2 files changed, 113 insertions(+), 82 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/components/legend_action.tsx diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 956cfec193636..3b9bc54c89003 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -4,19 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useState } from 'react'; -import { - EuiButtonIcon, - EuiFlexGroup, - EuiFlexItem, - EuiListGroup, - EuiListGroupItem, - EuiPanel, - EuiPopover, - EuiSpacer, - EuiTitle, - useEuiTheme, -} from '@elastic/eui'; +import React, { useCallback, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; import { Chart, Axis, @@ -24,7 +13,6 @@ import { Settings, ScaleType, niceTimeFormatter, - LegendActionProps, DARK_THEME, LIGHT_THEME, } from '@elastic/charts'; @@ -33,7 +21,7 @@ import { i18n } from '@kbn/i18n'; import { MetricsResponse } from '../types'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; import { MetricTypes } from '../../../common/rest_types'; -import { DatasetQualityLink } from './dataset_quality_link'; +import { LegendAction } from './legend_action'; interface ChartsProps { data: MetricsResponse; } @@ -57,29 +45,15 @@ export const Charts: React.FC = ({ data }) => { const theme = useEuiTheme(); const { services: { - share: { - url: { locators }, - }, application: { capabilities }, }, } = useKibanaContextForPlugin(); - const hasDataSetQualityFeature = capabilities.data_quality; - const hasIndexManagementFeature = capabilities.index_management; - - const onClickIndexManagement = async ({ dataStreamName }: { dataStreamName: string }) => { - // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 - const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); - if (dataQualityLocator) { - await dataQualityLocator.navigate({ - sectionId: 'data', - appId: `index_management/data_streams/${dataStreamName}`, - }); - } - }; + const hasDataSetQualityFeature = !!capabilities?.data_quality; + const hasIndexManagementFeature = !!capabilities?.index_management; - const togglePopover = (streamName: string) => { - setPopoverOpen(popoverOpen === streamName ? null : streamName); - }; + const togglePopover = useCallback((streamName: string | null) => { + setPopoverOpen((prev) => (prev === streamName ? null : streamName)); + }, []); return ( {data.charts.map((chart, idx) => { @@ -101,54 +75,16 @@ export const Charts: React.FC = ({ data }) => { showLegend={true} legendPosition="right" xDomain={{ min: minTimestamp, max: maxTimestamp }} - legendAction={({ label }: LegendActionProps) => { - const uniqueStreamName = `${idx}-${label}`; - return ( - - - - togglePopover(uniqueStreamName)} - /> - - - } - isOpen={popoverOpen === uniqueStreamName} - closePopover={() => setPopoverOpen(null)} - anchorPosition="downRight" - > - - { - navigator.clipboard.writeText(label); - }} - /> - - - {hasIndexManagementFeature && ( - - onClickIndexManagement({ - dataStreamName: label, - }) - } - /> - )} - {hasDataSetQualityFeature && ( - - )} - - - - ); - }} + legendAction={({ label }) => ( + + )} /> {chart.series.map((stream, streamIdx) => ( void; + hasIndexManagementFeature: boolean; + hasDataSetQualityFeature: boolean; + label: string; +} + +export const LegendAction: React.FC = React.memo( + ({ + label, + idx, + popoverOpen, + togglePopover, + hasIndexManagementFeature, + hasDataSetQualityFeature, + }) => { + const uniqueStreamName = `${idx}-${label}`; + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const onClickIndexManagement = useCallback(async () => { + // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 + const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); + if (dataQualityLocator) { + await dataQualityLocator.navigate({ + sectionId: 'data', + appId: `index_management/data_streams/${label}`, + }); + } + togglePopover(null); // Close the popover after action + }, [label, locators, togglePopover]); + + const onCopyDataStreamName = useCallback(() => { + navigator.clipboard.writeText(label); + togglePopover(null); // Close popover after copying + }, [label, togglePopover]); + + return ( + + + + togglePopover(uniqueStreamName)} + /> + + + } + isOpen={popoverOpen === uniqueStreamName} + closePopover={() => togglePopover(null)} + anchorPosition="downRight" + > + + + + + {hasIndexManagementFeature && ( + + )} + {hasDataSetQualityFeature && } + + + + ); + } +); From 15521118d6434cc050d1e0b7df46b0b819228722 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 13:44:39 -0400 Subject: [PATCH 19/53] clean up DatasetQualityLink component --- .../app/components/dataset_quality_link.tsx | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx index f095563a65a6e..d71093be9bb4a 100644 --- a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -5,12 +5,44 @@ * 2.0. */ -import { EuiListGroupItem } from '@elastic/eui'; - import React from 'react'; +import { EuiListGroupItem } from '@elastic/eui'; import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +interface DatasetQualityLinkProps { + dataStreamName: string; +} + +export const DatasetQualityLink: React.FC = React.memo( + ({ dataStreamName }) => { + const { + services: { + share: { url }, + }, + } = useKibanaContextForPlugin(); + + const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); + + const onClickDataQuality = async () => { + const locatorParams: DataQualityLocatorParams = { + filters: { + // TODO: get time range from our page state + timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + }, + }; + const dataset = getDatasetFromDataStream(dataStreamName); + if (locatorParams?.filters && dataset) { + locatorParams.filters.query = dataset; + } + if (locator) { + await locator.navigate(locatorParams); + } + }; + return ; + } +); + function getDatasetFromDataStream(dataStreamName: string): string | null { const parts = dataStreamName.split('-'); if (parts.length !== 3) { @@ -18,30 +50,3 @@ function getDatasetFromDataStream(dataStreamName: string): string | null { } return parts[1]; } - -export const DatasetQualityLink = React.memo(({ dataStreamName }: { dataStreamName: string }) => { - const { - services: { - share: { url }, - }, - } = useKibanaContextForPlugin(); - - const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); - const onClickDataQuality = async () => { - const locatorParams: DataQualityLocatorParams = { - filters: { - // TODO: get time range from our page state - timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, - }, - }; - const dataset = getDatasetFromDataStream(dataStreamName); - if (locatorParams?.filters && dataset) { - locatorParams.filters.query = dataset; - } - if (locator) { - await locator.navigate(locatorParams); - } - }; - - return onClickDataQuality()} />; -}); From b160b5938ce8b28aec46cb542149d3a50c6092b7 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 15:02:39 -0400 Subject: [PATCH 20/53] separate out components --- .../public/app/components/chart_panel.tsx | 121 ++++++++++++++++++ .../public/app/components/charts.tsx | 114 ++--------------- .../public/app/components/legend_action.tsx | 14 +- 3 files changed, 137 insertions(+), 112 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/components/chart_panel.tsx diff --git a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx new file mode 100644 index 0000000000000..e17ad2692c63a --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo } from 'react'; +import numeral from '@elastic/numeral'; +import { EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; +import { + Chart, + Axis, + BarSeries, + Settings, + ScaleType, + niceTimeFormatter, + DARK_THEME, + LIGHT_THEME, +} from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import { LegendAction } from './legend_action'; +import { MetricTypes } from '../../../common/rest_types'; +import { Chart as ChartData } from '../types'; + +// TODO: Remove this when we have a title for each metric type +type ChartKey = Extract; +export const chartKeyToTitleMap: Record = { + ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { + defaultMessage: 'Data Ingested', + }), + storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { + defaultMessage: 'Data Retained in Storage', + }), +}; + +interface ChartPanelProps { + data: ChartData; + idx: number; + popoverOpen: string | null; + togglePopover: (streamName: string | null) => void; +} + +export const ChartPanel: React.FC = ({ + data, + idx, + popoverOpen, + togglePopover, +}) => { + const theme = useEuiTheme(); + + const chartTimestamps = data.series.flatMap((series) => series.data.map((d) => d.x)); + + const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)]; + + const tickFormat = useMemo( + () => niceTimeFormatter([minTimestamp, maxTimestamp]), + [minTimestamp, maxTimestamp] + ); + + const renderLegendAction = useCallback( + ({ label }: { label: string }) => { + return ( + + ); + }, + [idx, popoverOpen, togglePopover] + ); + return ( + + + +
{chartKeyToTitleMap[data.key as ChartKey] || data.key}
+ + + + {data.series.map((stream, streamIdx) => ( + [point.x, point.y])} + xScaleType={ScaleType.Time} + yScaleType={ScaleType.Linear} + xAccessor={0} + yAccessors={[1]} + stackAccessors={[0]} + /> + ))} + + + + formatBytes(d)} + /> + + + + ); +}; +const formatBytes = (bytes: number) => { + return numeral(bytes).format('0.0 b'); +}; diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 3b9bc54c89003..ccf753d579f20 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -5,120 +5,30 @@ * 2.0. */ import React, { useCallback, useState } from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; -import { - Chart, - Axis, - BarSeries, - Settings, - ScaleType, - niceTimeFormatter, - DARK_THEME, - LIGHT_THEME, -} from '@elastic/charts'; -import numeral from '@elastic/numeral'; -import { i18n } from '@kbn/i18n'; +import { EuiFlexGroup } from '@elastic/eui'; import { MetricsResponse } from '../types'; -import { useKibanaContextForPlugin } from '../../utils/use_kibana'; -import { MetricTypes } from '../../../common/rest_types'; -import { LegendAction } from './legend_action'; +import { ChartPanel } from './chart_panel'; interface ChartsProps { data: MetricsResponse; } -const formatBytes = (bytes: number) => { - return numeral(bytes).format('0.0 b'); -}; - -// TODO: Remove this when we have a title for each metric type -type ChartKey = Extract; -export const chartKeyToTitleMap: Record = { - ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { - defaultMessage: 'Data Ingested', - }), - storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { - defaultMessage: 'Data Retained in Storage', - }), -}; export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); - const theme = useEuiTheme(); - const { - services: { - application: { capabilities }, - }, - } = useKibanaContextForPlugin(); - const hasDataSetQualityFeature = !!capabilities?.data_quality; - const hasIndexManagementFeature = !!capabilities?.index_management; - const togglePopover = useCallback((streamName: string | null) => { setPopoverOpen((prev) => (prev === streamName ? null : streamName)); }, []); + return ( - {data.charts.map((chart, idx) => { - const chartTimestamps = chart.series.flatMap((series) => series.data.map((d) => d.x)); - const minTimestamp = Math.min(...chartTimestamps); - const maxTimestamp = Math.max(...chartTimestamps); - const tickFormat = niceTimeFormatter([minTimestamp, maxTimestamp]); - - return ( - - -
- -
{chartKeyToTitleMap[chart.key] || chart.key}
- - - ( - - )} - /> - {chart.series.map((stream, streamIdx) => ( - [point.x, point.y])} - xScaleType={ScaleType.Time} - yScaleType={ScaleType.Linear} - xAccessor={0} - yAccessors={[1]} - stackAccessors={[0]} - /> - ))} - - - - formatBytes(d)} - /> - -
- - - ); - })} + {data.charts.map((chart, idx) => ( + + ))} ); }; diff --git a/x-pack/plugins/data_usage/public/app/components/legend_action.tsx b/x-pack/plugins/data_usage/public/app/components/legend_action.tsx index 611c8acf0f320..a816d1f8eadda 100644 --- a/x-pack/plugins/data_usage/public/app/components/legend_action.tsx +++ b/x-pack/plugins/data_usage/public/app/components/legend_action.tsx @@ -21,28 +21,22 @@ interface LegendActionProps { idx: number; popoverOpen: string | null; togglePopover: (streamName: string | null) => void; - hasIndexManagementFeature: boolean; - hasDataSetQualityFeature: boolean; label: string; } export const LegendAction: React.FC = React.memo( - ({ - label, - idx, - popoverOpen, - togglePopover, - hasIndexManagementFeature, - hasDataSetQualityFeature, - }) => { + ({ label, idx, popoverOpen, togglePopover }) => { const uniqueStreamName = `${idx}-${label}`; const { services: { share: { url: { locators }, }, + application: { capabilities }, }, } = useKibanaContextForPlugin(); + const hasDataSetQualityFeature = !!capabilities?.data_quality; + const hasIndexManagementFeature = !!capabilities?.index_management; const onClickIndexManagement = useCallback(async () => { // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 From 16a5efdf5d6270ff51f6769364f182cb313a2327 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 09:30:55 -0400 Subject: [PATCH 21/53] pass datepicker state to dataset quality link --- .../public/app/components/dataset_quality_link.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx index d71093be9bb4a..00f9ea1a1e6a8 100644 --- a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -9,6 +9,7 @@ import React from 'react'; import { EuiListGroupItem } from '@elastic/eui'; import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +import { useDateRangePicker } from '../hooks/use_date_picker'; interface DatasetQualityLinkProps { dataStreamName: string; @@ -16,19 +17,18 @@ interface DatasetQualityLinkProps { export const DatasetQualityLink: React.FC = React.memo( ({ dataStreamName }) => { + const { dateRangePickerState } = useDateRangePicker(); const { services: { share: { url }, }, } = useKibanaContextForPlugin(); - + const { startDate, endDate } = dateRangePickerState; const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); - const onClickDataQuality = async () => { const locatorParams: DataQualityLocatorParams = { filters: { - // TODO: get time range from our page state - timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + timeRange: { from: startDate, to: endDate, refresh: { pause: true, value: 0 } }, }, }; const dataset = getDatasetFromDataStream(dataStreamName); From 21c3af3736334cc73ff962c262575f7e24971b6e Mon Sep 17 00:00:00 2001 From: neptunian Date: Tue, 24 Sep 2024 15:34:35 -0400 Subject: [PATCH 22/53] add ui components to data_usage and mock data --- x-pack/plugins/data_usage/common/types.ts | 9 + x-pack/plugins/data_usage/kibana.jsonc | 3 + .../public/app/components/charts.tsx | 87 +++++++++ .../public/app/components/date_picker.tsx | 52 +++++ .../data_usage/public/app/data_usage.tsx | 181 ++++++++++++++++++ .../public/app/hooks/use_date_picker.tsx | 29 +++ x-pack/plugins/data_usage/public/app/types.ts | 25 +++ .../plugins/data_usage/public/application.tsx | 3 +- .../public/utils/use_breadcrumbs.tsx | 28 +++ 9 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/data_usage/common/types.ts create mode 100644 x-pack/plugins/data_usage/public/app/components/charts.tsx create mode 100644 x-pack/plugins/data_usage/public/app/components/date_picker.tsx create mode 100644 x-pack/plugins/data_usage/public/app/data_usage.tsx create mode 100644 x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx create mode 100644 x-pack/plugins/data_usage/public/app/types.ts create mode 100644 x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx diff --git a/x-pack/plugins/data_usage/common/types.ts b/x-pack/plugins/data_usage/common/types.ts new file mode 100644 index 0000000000000..d80bae2458d09 --- /dev/null +++ b/x-pack/plugins/data_usage/common/types.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// temporary type until agreed on +export type MetricKey = 'ingestedMax' | 'retainedMax'; diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 9b0f2d193925e..82411d169ef0c 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -12,5 +12,8 @@ "requiredBundles": [ "kibanaReact", ], + "extraPublicDirs": [ + "common", + ] } } diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx new file mode 100644 index 0000000000000..f4d6cb3883245 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; +import { Chart, Axis, BarSeries, Settings, ScaleType, niceTimeFormatter } from '@elastic/charts'; +import numeral from '@elastic/numeral'; +import { i18n } from '@kbn/i18n'; +import { MetricsResponse } from '../types'; +import { MetricKey } from '../../../common/types'; +interface ChartsProps { + data: MetricsResponse; +} +const formatBytes = (bytes: number) => { + return numeral(bytes).format('0.0 b'); +}; + +export const chartKeyToTitleMap: Record = { + ingestedMax: i18n.translate('xpack.dataUsage.charts.ingestedMax', { + defaultMessage: 'Data Ingested', + }), + retainedMax: i18n.translate('xpack.dataUsage.charts.retainedMax', { + defaultMessage: 'Data Retained in Storage', + }), +}; + +export const Charts: React.FC = ({ data }) => { + return ( + + {data.charts.map((chart, idx) => { + const chartTimestamps = chart.series.flatMap((series) => series.data.map((d) => d.x)); + const minTimestamp = Math.min(...chartTimestamps); + const maxTimestamp = Math.max(...chartTimestamps); + const tickFormat = niceTimeFormatter([minTimestamp, maxTimestamp]); + + return ( + + +
+ +
{chartKeyToTitleMap[chart.key] || chart.key}
+ + + + {chart.series.map((stream, streamIdx) => ( + [point.x, point.y])} + xScaleType={ScaleType.Time} + yScaleType={ScaleType.Linear} + xAccessor={0} + yAccessors={[1]} + stackAccessors={[0]} + /> + ))} + + + + formatBytes(d)} + /> + +
+ + + ); + })} + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx new file mode 100644 index 0000000000000..f95af6d6bfc43 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiSuperDatePicker } from '@elastic/eui'; +import { useDatePickerContext } from '../hooks/use_date_picker'; + +export const DatePicker: React.FC = () => { + const { + startDate, + endDate, + isAutoRefreshActive, + refreshInterval, + setStartDate, + setEndDate, + setIsAutoRefreshActive, + setRefreshInterval, + } = useDatePickerContext(); + + const onTimeChange = ({ start, end }: { start: string; end: string }) => { + setStartDate(start); + setEndDate(end); + // Trigger API call or data refresh here + }; + + const onRefreshChange = ({ + isPaused, + refreshInterval: onRefreshRefreshInterval, + }: { + isPaused: boolean; + refreshInterval: number; + }) => { + setIsAutoRefreshActive(!isPaused); + setRefreshInterval(onRefreshRefreshInterval); + }; + + return ( + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx new file mode 100644 index 0000000000000..3c80854bf9cfe --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { Charts } from './components/charts'; +import { DatePicker } from './components/date_picker'; +import { MetricsResponse } from './types'; +import { useBreadcrumbs } from '../utils/use_breadcrumbs'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; +import { PLUGIN_NAME } from '../../common'; +import { DatePickerProvider } from './hooks/use_date_picker'; + +const response: MetricsResponse = { + charts: [ + { + key: 'ingestedMax', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 13756849 }, + { x: 1726862130000, y: 14657904 }, + { x: 1726865730000, y: 12798561 }, + { x: 1726869330000, y: 13578213 }, + { x: 1726872930000, y: 14123495 }, + { x: 1726876530000, y: 13876548 }, + { x: 1726880130000, y: 12894561 }, + { x: 1726883730000, y: 14478953 }, + { x: 1726887330000, y: 14678905 }, + { x: 1726890930000, y: 13976547 }, + { x: 1726894530000, y: 14568945 }, + { x: 1726898130000, y: 13789561 }, + { x: 1726901730000, y: 14478905 }, + { x: 1726905330000, y: 13956423 }, + { x: 1726908930000, y: 14598234 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + ], + }, + { + key: 'retainedMax', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + // Repeat similar structure for more data streams... + ], + }, + ], +}; + +export const DataUsage = () => { + const { + services: { chrome, appParams }, + } = useKibanaContextForPlugin(); + + useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); + + return ( + + +

+ {i18n.translate('xpack.dataUsage.pageTitle', { + defaultMessage: 'Data usage stats', + })} +

+ + + + + + + + + ; + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx new file mode 100644 index 0000000000000..cd8ac35c98839 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState } from 'react'; +import createContainer from 'constate'; + +const useDatePicker = () => { + const [startDate, setStartDate] = useState('now-24h'); + const [endDate, setEndDate] = useState('now'); + const [isAutoRefreshActive, setIsAutoRefreshActive] = useState(false); + const [refreshInterval, setRefreshInterval] = useState(0); + + return { + startDate, + setStartDate, + endDate, + setEndDate, + isAutoRefreshActive, + setIsAutoRefreshActive, + refreshInterval, + setRefreshInterval, + }; +}; + +export const [DatePickerProvider, useDatePickerContext] = createContainer(useDatePicker); diff --git a/x-pack/plugins/data_usage/public/app/types.ts b/x-pack/plugins/data_usage/public/app/types.ts new file mode 100644 index 0000000000000..279a7e0e863ef --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { MetricKey } from '../../common/types'; + +export interface DataPoint { + x: number; + y: number; +} + +export interface Series { + streamName: string; + data: DataPoint[]; +} +export interface Chart { + key: MetricKey; + series: Series[]; +} + +export interface MetricsResponse { + charts: Chart[]; +} diff --git a/x-pack/plugins/data_usage/public/application.tsx b/x-pack/plugins/data_usage/public/application.tsx index 1e6c35c4b8f0a..a160a5742d36d 100644 --- a/x-pack/plugins/data_usage/public/application.tsx +++ b/x-pack/plugins/data_usage/public/application.tsx @@ -16,6 +16,7 @@ import { PerformanceContextProvider } from '@kbn/ebt-tools'; import { useKibanaContextForPluginProvider } from './utils/use_kibana'; import { DataUsageStartDependencies, DataUsagePublicStart } from './types'; import { PLUGIN_ID } from '../common'; +import { DataUsage } from './app/data_usage'; export const renderApp = ( core: CoreStart, @@ -51,7 +52,7 @@ const AppWithExecutionContext = ({ -
Data Usage
} /> + diff --git a/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx b/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx new file mode 100644 index 0000000000000..928ee73ad5280 --- /dev/null +++ b/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ChromeBreadcrumb, ChromeStart } from '@kbn/core-chrome-browser'; + +import { useEffect } from 'react'; +import { ManagementAppMountParams } from '@kbn/management-plugin/public'; + +export const useBreadcrumbs = ( + breadcrumbs: ChromeBreadcrumb[], + params: ManagementAppMountParams, + chromeService: ChromeStart +) => { + const { docTitle } = chromeService; + const isMultiple = breadcrumbs.length > 1; + + const docTitleValue = isMultiple ? breadcrumbs[breadcrumbs.length - 1].text : breadcrumbs[0].text; + + docTitle.change(docTitleValue as string); + + useEffect(() => { + params.setBreadcrumbs(breadcrumbs); + }, [breadcrumbs, params]); +}; From 48231a399ab09c1e2bef497b9c449840d0d0e141 Mon Sep 17 00:00:00 2001 From: neptunian Date: Wed, 25 Sep 2024 07:55:22 -0400 Subject: [PATCH 23/53] remove deprecated extraPublicDirs --- x-pack/plugins/data_usage/kibana.jsonc | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 82411d169ef0c..9b0f2d193925e 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -12,8 +12,5 @@ "requiredBundles": [ "kibanaReact", ], - "extraPublicDirs": [ - "common", - ] } } From d0b8bcaceaf355f9ea76099f41af1b55587015bf Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 30 Sep 2024 13:55:33 -0400 Subject: [PATCH 24/53] update icon card to stats --- packages/kbn-management/cards_navigation/src/consts.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-management/cards_navigation/src/consts.tsx b/packages/kbn-management/cards_navigation/src/consts.tsx index 16e655c5510ad..6a22b9e33d620 100644 --- a/packages/kbn-management/cards_navigation/src/consts.tsx +++ b/packages/kbn-management/cards_navigation/src/consts.tsx @@ -77,7 +77,7 @@ export const appDefinitions: Record = { description: i18n.translate('management.landing.withCardNavigation.dataUsageDescription', { defaultMessage: 'View data usage and retention.', }), - icon: 'documents', + icon: 'stats', }, [AppIds.RULES]: { From bae611724b09775316e2a6bbc1d3cb0113ee3187 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 30 Sep 2024 15:55:54 -0400 Subject: [PATCH 25/53] data stream action menu --- .../public/app/components/charts.tsx | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index f4d6cb3883245..3308e5b15304c 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -4,9 +4,26 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; -import { Chart, Axis, BarSeries, Settings, ScaleType, niceTimeFormatter } from '@elastic/charts'; +import React, { useState } from 'react'; +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiListGroup, + EuiListGroupItem, + EuiPanel, + EuiPopover, + EuiTitle, +} from '@elastic/eui'; +import { + Chart, + Axis, + BarSeries, + Settings, + ScaleType, + niceTimeFormatter, + LegendActionProps, +} from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; import { MetricsResponse } from '../types'; @@ -28,6 +45,11 @@ export const chartKeyToTitleMap: Record = { }; export const Charts: React.FC = ({ data }) => { + const [popoverOpen, setPopoverOpen] = useState(null); + + const togglePopover = (streamName: string) => { + setPopoverOpen(popoverOpen === streamName ? null : streamName); + }; return ( {data.charts.map((chart, idx) => { @@ -48,6 +70,47 @@ export const Charts: React.FC = ({ data }) => { showLegend={true} legendPosition="right" xDomain={{ min: minTimestamp, max: maxTimestamp }} + legendAction={({ label }: LegendActionProps) => { + const streamName = `${idx}-${label}`; + return ( + + + + togglePopover(streamName)} + /> + + + } + isOpen={popoverOpen === streamName} + closePopover={() => setPopoverOpen(null)} + anchorPosition="downRight" + > + + undefined} + /> + undefined} + /> + undefined} + /> + + + + ); + }} /> {chart.series.map((stream, streamIdx) => ( Date: Thu, 3 Oct 2024 07:06:17 -0400 Subject: [PATCH 26/53] add elasticsearch feature privilege --- x-pack/plugins/data_usage/server/plugin.ts | 15 ++++++++++++++- x-pack/plugins/data_usage/server/types.ts | 7 +++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/data_usage/server/plugin.ts b/x-pack/plugins/data_usage/server/plugin.ts index 8ab49d5104fff..d4309d5742528 100644 --- a/x-pack/plugins/data_usage/server/plugin.ts +++ b/x-pack/plugins/data_usage/server/plugin.ts @@ -14,6 +14,7 @@ import type { DataUsageSetupDependencies, DataUsageStartDependencies, } from './types'; +import { PLUGIN_ID } from '../common'; export class DataUsagePlugin implements @@ -28,7 +29,19 @@ export class DataUsagePlugin constructor(context: PluginInitializerContext) { this.logger = context.logger.get(); } - setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup { + setup(coreSetup: CoreSetup, { features }: DataUsageSetupDependencies): DataUsageServerSetup { + features.registerElasticsearchFeature({ + id: PLUGIN_ID, + management: { + data: [PLUGIN_ID], + }, + privileges: [ + { + requiredClusterPrivileges: ['monitor'], + ui: [], + }, + ], + }); return {}; } diff --git a/x-pack/plugins/data_usage/server/types.ts b/x-pack/plugins/data_usage/server/types.ts index 9f43ae2d3c298..fa0d561701a9d 100644 --- a/x-pack/plugins/data_usage/server/types.ts +++ b/x-pack/plugins/data_usage/server/types.ts @@ -5,10 +5,13 @@ * 2.0. */ -/* eslint-disable @typescript-eslint/no-empty-interface*/ +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; -export interface DataUsageSetupDependencies {} +export interface DataUsageSetupDependencies { + features: FeaturesPluginSetup; +} +/* eslint-disable @typescript-eslint/no-empty-interface*/ export interface DataUsageStartDependencies {} export interface DataUsageServerSetup {} From dbd912055bf90ec576dcf899b87b6bf85b986cc0 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 08:57:58 -0400 Subject: [PATCH 27/53] add descriptive text and fix layout spacing --- .../data_usage/public/app/data_usage.tsx | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 3c80854bf9cfe..656e5e81ebc5d 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -6,8 +6,9 @@ */ import React from 'react'; -import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem, EuiText, EuiPageSection } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import { Charts } from './components/charts'; import { DatePicker } from './components/date_picker'; import { MetricsResponse } from './types'; @@ -169,13 +170,23 @@ export const DataUsage = () => {

- - - - - - - ; + + + + + + + + + + + + + ; + ); }; From d42614f4664f1f3cd6632a24c2848b38f04c85d6 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 09:36:05 -0400 Subject: [PATCH 28/53] fix chart theme dark mode --- x-pack/plugins/data_usage/public/app/components/charts.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 3308e5b15304c..3e74acc685c9a 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -14,6 +14,7 @@ import { EuiPanel, EuiPopover, EuiTitle, + useEuiTheme, } from '@elastic/eui'; import { Chart, @@ -23,9 +24,12 @@ import { ScaleType, niceTimeFormatter, LegendActionProps, + DARK_THEME, + LIGHT_THEME, } from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; + import { MetricsResponse } from '../types'; import { MetricKey } from '../../../common/types'; interface ChartsProps { @@ -46,6 +50,7 @@ export const chartKeyToTitleMap: Record = { export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); + const theme = useEuiTheme(); const togglePopover = (streamName: string) => { setPopoverOpen(popoverOpen === streamName ? null : streamName); @@ -67,6 +72,7 @@ export const Charts: React.FC = ({ data }) => { Date: Thu, 3 Oct 2024 11:37:16 -0400 Subject: [PATCH 29/53] fix type --- x-pack/plugins/data_usage/public/app/data_usage.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 656e5e81ebc5d..d97376252cd1d 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -6,7 +6,14 @@ */ import React from 'react'; -import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem, EuiText, EuiPageSection } from '@elastic/eui'; +import { + EuiTitle, + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiPageSection, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { Charts } from './components/charts'; From 9a02d58656a726e7d6c4cb080efee218af153f0f Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 15:29:01 -0400 Subject: [PATCH 30/53] add data quality set locator --- .../observability/locators/dataset_quality.ts | 1 + .../public/app/components/charts.tsx | 59 +++++++++++++++---- .../data_usage/public/app/data_usage.tsx | 2 +- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/packages/deeplinks/observability/locators/dataset_quality.ts b/packages/deeplinks/observability/locators/dataset_quality.ts index 9a6dd85ade2d2..6ae42f3805381 100644 --- a/packages/deeplinks/observability/locators/dataset_quality.ts +++ b/packages/deeplinks/observability/locators/dataset_quality.ts @@ -27,6 +27,7 @@ type TimeRangeConfig = { // eslint-disable-next-line @typescript-eslint/consistent-type-definitions type Filters = { timeRange: TimeRangeConfig; + query?: string; }; export interface DataQualityLocatorParams extends SerializableRecord { diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 3e74acc685c9a..08f51f60f887c 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -29,9 +29,10 @@ import { } from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; - +import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { MetricsResponse } from '../types'; import { MetricKey } from '../../../common/types'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; interface ChartsProps { data: MetricsResponse; } @@ -47,10 +48,41 @@ export const chartKeyToTitleMap: Record = { defaultMessage: 'Data Retained in Storage', }), }; - +function getDatasetFromDataStream(dataStreamName: string): string | null { + const parts = dataStreamName.split('-'); + if (parts.length !== 3) { + return null; + } + return parts[1]; +} export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); const theme = useEuiTheme(); + const { + services: { + share: { + url: { locators }, + }, + application: { capabilities }, + }, + } = useKibanaContextForPlugin(); + const hasDataSetQualityFeature = capabilities.data_quality; + const onClickDataQuality = async ({ dataStreamName }: { dataStreamName: string }) => { + const dataQualityLocator = locators.get(DATA_QUALITY_LOCATOR_ID); + const locatorParams: DataQualityLocatorParams = { + filters: { + // TODO: get time range from our page state + timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + }, + }; + const dataset = getDatasetFromDataStream(dataStreamName); + if (locatorParams?.filters && dataset) { + locatorParams.filters.query = dataset; + } + if (dataQualityLocator) { + await dataQualityLocator.navigate(locatorParams); + } + }; const togglePopover = (streamName: string) => { setPopoverOpen(popoverOpen === streamName ? null : streamName); @@ -64,7 +96,7 @@ export const Charts: React.FC = ({ data }) => { const tickFormat = niceTimeFormatter([minTimestamp, maxTimestamp]); return ( - +
@@ -77,7 +109,7 @@ export const Charts: React.FC = ({ data }) => { legendPosition="right" xDomain={{ min: minTimestamp, max: maxTimestamp }} legendAction={({ label }: LegendActionProps) => { - const streamName = `${idx}-${label}`; + const uniqueStreamName = `${idx}-${label}`; return ( = ({ data }) => { togglePopover(streamName)} + onClick={() => togglePopover(uniqueStreamName)} /> } - isOpen={popoverOpen === streamName} + isOpen={popoverOpen === uniqueStreamName} closePopover={() => setPopoverOpen(null)} anchorPosition="downRight" > @@ -107,11 +139,16 @@ export const Charts: React.FC = ({ data }) => { label="Manage data stream" onClick={() => undefined} /> - undefined} - /> + {hasDataSetQualityFeature && ( + + onClickDataQuality({ + dataStreamName: label, + }) + } + /> + )} diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index d97376252cd1d..a43f18f0aa30a 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -192,7 +192,7 @@ export const DataUsage = () => { - ; + ); From 70e568318b7d8cd77ac62387019a70f686d3696c Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 3 Oct 2024 19:49:38 -0400 Subject: [PATCH 31/53] fix title --- x-pack/plugins/data_usage/public/app/data_usage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index a43f18f0aa30a..476caaa8930a7 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -172,7 +172,7 @@ export const DataUsage = () => {

{i18n.translate('xpack.dataUsage.pageTitle', { - defaultMessage: 'Data usage stats', + defaultMessage: 'Data Usage', })}

From accb18a38889a53dd3fe9a7656e0db756a5d3e53 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 25 Sep 2024 12:17:35 +0200 Subject: [PATCH 32/53] initial internal API setup --- .../data_usage/server/common/errors.ts | 18 ++ x-pack/plugins/data_usage/server/config.ts | 13 +- x-pack/plugins/data_usage/server/index.ts | 8 +- x-pack/plugins/data_usage/server/plugin.ts | 33 +++- .../data_usage/server/routes/error_handler.ts | 40 ++++ .../data_usage/server/routes/index.tsx | 16 ++ .../server/routes/internal/index.tsx | 7 + .../server/routes/internal/usage_metrics.ts | 40 ++++ .../routes/internal/usage_metrics_handler.ts | 139 ++++++++++++++ x-pack/plugins/data_usage/server/types.ts | 19 -- .../plugins/data_usage/server/types/index.ts | 9 + .../server/types/rest_types/index.ts | 8 + .../types/rest_types/usage_metrics.test.ts | 174 ++++++++++++++++++ .../server/types/rest_types/usage_metrics.ts | 104 +++++++++++ .../plugins/data_usage/server/types/types.ts | 42 +++++ .../server/utils/custom_http_request_error.ts | 22 +++ .../plugins/data_usage/server/utils/index.ts | 9 + .../server/utils/validate_time_range.ts | 17 ++ 18 files changed, 687 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugins/data_usage/server/common/errors.ts create mode 100644 x-pack/plugins/data_usage/server/routes/error_handler.ts create mode 100644 x-pack/plugins/data_usage/server/routes/index.tsx create mode 100644 x-pack/plugins/data_usage/server/routes/internal/index.tsx create mode 100644 x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts create mode 100644 x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts delete mode 100644 x-pack/plugins/data_usage/server/types.ts create mode 100644 x-pack/plugins/data_usage/server/types/index.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/index.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts create mode 100644 x-pack/plugins/data_usage/server/types/types.ts create mode 100644 x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts create mode 100644 x-pack/plugins/data_usage/server/utils/index.ts create mode 100644 x-pack/plugins/data_usage/server/utils/validate_time_range.ts diff --git a/x-pack/plugins/data_usage/server/common/errors.ts b/x-pack/plugins/data_usage/server/common/errors.ts new file mode 100644 index 0000000000000..7a43a10108be1 --- /dev/null +++ b/x-pack/plugins/data_usage/server/common/errors.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class BaseError extends Error { + constructor(message: string, public readonly meta?: MetaType) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } + } +} diff --git a/x-pack/plugins/data_usage/server/config.ts b/x-pack/plugins/data_usage/server/config.ts index 6453cce4f4d56..bf89431f2abea 100644 --- a/x-pack/plugins/data_usage/server/config.ts +++ b/x-pack/plugins/data_usage/server/config.ts @@ -6,9 +6,18 @@ */ import { schema, type TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from '@kbn/core/server'; -export const config = schema.object({ +export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: false }), }); -export type DataUsageConfig = TypeOf; +export type DataUsageConfigType = TypeOf; + +export const createConfig = (context: PluginInitializerContext): DataUsageConfigType => { + const pluginConfig = context.config.get>(); + + return { + ...pluginConfig, + }; +}; diff --git a/x-pack/plugins/data_usage/server/index.ts b/x-pack/plugins/data_usage/server/index.ts index 3aa49a184d003..66d839303d716 100644 --- a/x-pack/plugins/data_usage/server/index.ts +++ b/x-pack/plugins/data_usage/server/index.ts @@ -9,7 +9,7 @@ import type { PluginInitializerContext, PluginConfigDescriptor, } from '@kbn/core/server'; -import { DataUsageConfig } from './config'; +import { DataUsageConfigType } from './config'; import { DataUsagePlugin } from './plugin'; import type { @@ -19,11 +19,11 @@ import type { DataUsageStartDependencies, } from './types'; -import { config as configSchema } from './config'; +import { configSchema } from './config'; export type { DataUsageServerSetup, DataUsageServerStart }; -export const config: PluginConfigDescriptor = { +export const config: PluginConfigDescriptor = { schema: configSchema, }; @@ -32,5 +32,5 @@ export const plugin: PluginInitializer< DataUsageServerStart, DataUsageSetupDependencies, DataUsageStartDependencies -> = async (pluginInitializerContext: PluginInitializerContext) => +> = async (pluginInitializerContext: PluginInitializerContext) => await new DataUsagePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/data_usage/server/plugin.ts b/x-pack/plugins/data_usage/server/plugin.ts index d4309d5742528..2beae9b22bba9 100644 --- a/x-pack/plugins/data_usage/server/plugin.ts +++ b/x-pack/plugins/data_usage/server/plugin.ts @@ -7,13 +7,16 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; -import { DataUsageConfig } from './config'; +import { DataUsageConfigType, createConfig } from './config'; import type { + DataUsageContext, + DataUsageRequestHandlerContext, DataUsageServerSetup, DataUsageServerStart, DataUsageSetupDependencies, DataUsageStartDependencies, } from './types'; +import { registerDataUsageRoutes } from './routes'; import { PLUGIN_ID } from '../common'; export class DataUsagePlugin @@ -25,12 +28,25 @@ export class DataUsagePlugin DataUsageStartDependencies > { - logger: Logger; - constructor(context: PluginInitializerContext) { + private readonly logger: Logger; + private dataUsageContext: DataUsageContext; + + constructor(context: PluginInitializerContext) { + const serverConfig = createConfig(context); + this.logger = context.logger.get(); + + this.logger.debug('data usage plugin initialized'); + this.dataUsageContext = { + logFactory: context.logger, + get serverConfig() { + return serverConfig; + }, + }; } - setup(coreSetup: CoreSetup, { features }: DataUsageSetupDependencies): DataUsageServerSetup { - features.registerElasticsearchFeature({ + setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup { + this.logger.debug('data usage plugin setup'); + pluginsSetup.features.registerElasticsearchFeature({ id: PLUGIN_ID, management: { data: [PLUGIN_ID], @@ -42,6 +58,9 @@ export class DataUsagePlugin }, ], }); + const router = coreSetup.http.createRouter(); + registerDataUsageRoutes(router, this.dataUsageContext); + return {}; } @@ -49,5 +68,7 @@ export class DataUsagePlugin return {}; } - public stop() {} + public stop() { + this.logger.debug('Stopping data usage plugin'); + } } diff --git a/x-pack/plugins/data_usage/server/routes/error_handler.ts b/x-pack/plugins/data_usage/server/routes/error_handler.ts new file mode 100644 index 0000000000000..7bd774f2e71e5 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/error_handler.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; +import { CustomHttpRequestError } from '../utils/custom_http_request_error'; +import { BaseError } from '../common/errors'; + +export class NotFoundError extends BaseError {} + +/** + * Default Endpoint Routes error handler + * @param logger + * @param res + * @param error + */ +export const errorHandler = ( + logger: Logger, + res: KibanaResponseFactory, + error: E +): IKibanaResponse => { + logger.error(error); + + if (error instanceof CustomHttpRequestError) { + return res.customError({ + statusCode: error.statusCode, + body: error, + }); + } + + if (error instanceof NotFoundError) { + return res.notFound({ body: error }); + } + + // Kibana CORE will take care of `500` errors when the handler `throw`'s, including logging the error + throw error; +}; diff --git a/x-pack/plugins/data_usage/server/routes/index.tsx b/x-pack/plugins/data_usage/server/routes/index.tsx new file mode 100644 index 0000000000000..b2d10b0ab53bc --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/index.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataUsageContext, DataUsageRouter } from '../types'; +import { registerUsageMetricsRoute } from './internal'; + +export const registerDataUsageRoutes = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + registerUsageMetricsRoute(router, dataUsageContext); +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/index.tsx b/x-pack/plugins/data_usage/server/routes/internal/index.tsx new file mode 100644 index 0000000000000..d623dd8dd0ef7 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/index.tsx @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { registerUsageMetricsRoute } from './usage_metrics'; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts new file mode 100644 index 0000000000000..439b0d4561f12 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + DataUsageContext, + DataUsageRouter, + UsageMetricsRequestSchema, + UsageMetricsResponseSchema, +} from '../../types'; + +import { getUsageMetricsHandler } from './usage_metrics_handler'; + +export const registerUsageMetricsRoute = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + if (dataUsageContext.serverConfig.enabled) { + router.versioned + .get({ + access: 'internal', + path: '/internal/api/data_usage/metrics', + }) + .addVersion( + { + version: '1', + validate: { + request: UsageMetricsRequestSchema, + response: { + 200: UsageMetricsResponseSchema, + }, + }, + }, + getUsageMetricsHandler(dataUsageContext) + ); + } +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts new file mode 100644 index 0000000000000..6369b13aadcc4 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandler } from '@kbn/core/server'; +import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; +import { + DataUsageContext, + DataUsageRequestHandlerContext, + UsageMetricsRequestSchemaQueryParams, +} from '../../types'; + +import { errorHandler } from '../error_handler'; + +export const getUsageMetricsHandler = ( + dataUsageContext: DataUsageContext +): RequestHandler< + never, + UsageMetricsRequestSchemaQueryParams, + unknown, + DataUsageRequestHandlerContext +> => { + const logger = dataUsageContext.logFactory.get('usageMetricsRoute'); + + return async (context, request, response) => { + try { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + + // @ts-ignore + const { from, to, metricTypes, dataStreams: dsNames, size } = request.query; + logger.debug(`Retrieving usage metrics`); + + const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse = + await esClient.indices.getDataStream({ + name: '*', + expand_wildcards: 'all', + }); + + const hasDataStreams = dataStreamsResponse.length > 0; + let userDsNames: string[] = []; + + if (dsNames?.length) { + userDsNames = typeof dsNames === 'string' ? [dsNames] : dsNames; + } else if (!userDsNames.length && hasDataStreams) { + userDsNames = dataStreamsResponse.map((ds) => ds.name); + } + + // If no data streams are found, return an empty response + if (!userDsNames.length) { + return response.ok({ + body: { + charts: [], + }, + }); + } + // TODO: fetch data from autoOps using userDsNames + + // mock data + const charts = [ + { + key: 'ingest_rate', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 13756849 }, + { x: 1726862130000, y: 14657904 }, + { x: 1726865730000, y: 12798561 }, + { x: 1726869330000, y: 13578213 }, + { x: 1726872930000, y: 14123495 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + ], + }, + { + streamName: 'data_stream_3', + data: [{ x: 1726858530000, y: 12576413 }], + }, + ], + }, + { + key: 'storage_retained', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + ], + }, + ], + }, + ]; + return response.ok({ + body: { + charts, + }, + }); + } catch (error) { + return errorHandler(logger, response, error); + } + }; +}; diff --git a/x-pack/plugins/data_usage/server/types.ts b/x-pack/plugins/data_usage/server/types.ts deleted file mode 100644 index fa0d561701a9d..0000000000000 --- a/x-pack/plugins/data_usage/server/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; - -export interface DataUsageSetupDependencies { - features: FeaturesPluginSetup; -} - -/* eslint-disable @typescript-eslint/no-empty-interface*/ -export interface DataUsageStartDependencies {} - -export interface DataUsageServerSetup {} - -export interface DataUsageServerStart {} diff --git a/x-pack/plugins/data_usage/server/types/index.ts b/x-pack/plugins/data_usage/server/types/index.ts new file mode 100644 index 0000000000000..96577c72a8116 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './rest_types'; +export * from './types'; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/index.ts b/x-pack/plugins/data_usage/server/types/rest_types/index.ts new file mode 100644 index 0000000000000..2a6dc5006d53e --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './usage_metrics'; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts new file mode 100644 index 0000000000000..de46215a70d85 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UsageMetricsRequestSchema } from './usage_metrics'; + +describe('usage_metrics schemas', () => { + it('should accept valid request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + }) + ).not.toThrow(); + }); + + it('should accept a single `metricTypes` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'ingest_rate', + }) + ).not.toThrow(); + }); + + it('should accept multiple `metricTypes` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['ingest_rate', 'storage_retained', 'index_rate'], + }) + ).not.toThrow(); + }); + + it('should accept a single string as `dataStreams` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'storage_retained', + dataStreams: 'data_stream_1', + }) + ).not.toThrow(); + }); + + it('should accept valid `size`', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 100, + }) + ).not.toThrow(); + }); + + it('should accept `dataStream` list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 3, + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], + }) + ).not.toThrow(); + }); + + it('should error if `dataStream` list is empty', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 3, + dataStreams: [], + }) + ).toThrowError('expected value of type [string] but got [Array]'); + }); + + it('should error if `dataStream` is given an empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 1, + dataStreams: ' ', + }) + ).toThrow('[dataStreams] must have at least one value'); + }); + + it('should error if `dataStream` is given an empty item in the list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + size: 3, + dataStreams: ['ds_1', ' '], + }) + ).toThrow('[dataStreams] list can not contain empty values'); + }); + + it('should error if `metricTypes` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ' ', + }) + ).toThrow(); + }); + + it('should error if `metricTypes` is empty item', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: [' ', 'storage_retained'], + }) + ).toThrow('[metricTypes] list can not contain empty values'); + }); + + it('should error if `metricTypes` is not a valid value', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'foo', + }) + ).toThrow( + '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' + ); + }); + + it('should error if `metricTypes` is not a valid list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow( + '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' + ); + }); + + it('should error if `from` is not valid', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: 'bar', + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + }) + ).toThrow('[from]: Invalid date'); + }); + + it('should error if `to` is not valid', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: 'foo', + metricTypes: ['storage_retained'], + }) + ).toThrow('[to]: Invalid date'); + }); +}); diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts new file mode 100644 index 0000000000000..5fe81567d94dc --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; + +const METRIC_TYPE_VALUES = [ + 'storage_retained', + 'ingest_rate', + 'search_vcu', + 'ingest_vcu', + 'ml_vcu', + 'index_latency', + 'index_rate', + 'search_latency', + 'search_rate', +] as const; + +// @ts-ignore +const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); + +const metricTypesSchema = { + // @ts-expect-error TS2769: No overload matches this call + schema: schema.oneOf(METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType))), + options: { minSize: 1, maxSize: METRIC_TYPE_VALUES.length }, +}; + +export const UsageMetricsRequestSchema = { + query: schema.object({ + from: schema.string({ + validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), + }), + to: schema.string({ + validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), + }), + size: schema.maybe(schema.number()), // should be same as dataStreams.length + metricTypes: schema.oneOf([ + schema.arrayOf(schema.string(), { + ...metricTypesSchema.options, + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[metricTypes] list can not contain empty values'; + } else if (values.map((v) => v.trim()).some((v) => !isValidMetricType(v))) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, + }), + schema.string({ + validate: (v) => { + if (!v.trim().length) { + return '[metricTypes] must have at least one value'; + } else if (!isValidMetricType(v)) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, + }), + ]), + dataStreams: schema.maybe( + schema.oneOf([ + schema.arrayOf(schema.string(), { + minSize: 1, + maxSize: 50, // TBD + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[dataStreams] list can not contain empty values'; + } + }, + }), + schema.string({ + validate: (v) => + v.trim().length ? undefined : '[dataStreams] must have at least one value', + }), + ]) + ), + }), +}; + +export type UsageMetricsRequestSchemaQueryParams = TypeOf; + +export const UsageMetricsResponseSchema = { + body: () => + schema.object({ + charts: schema.arrayOf( + schema.object({ + key: metricTypesSchema.schema, + series: schema.arrayOf( + schema.object({ + streamName: schema.string(), + data: schema.arrayOf( + schema.object({ + x: schema.number(), + y: schema.number(), + }) + ), + }) + ), + }), + { maxSize: 2 } + ), + }), +}; diff --git a/x-pack/plugins/data_usage/server/types/types.ts b/x-pack/plugins/data_usage/server/types/types.ts new file mode 100644 index 0000000000000..c90beb184f020 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + CoreRequestHandlerContext, + CustomRequestHandlerContext, + IRouter, + LoggerFactory, +} from '@kbn/core/server'; +import { DeepReadonly } from 'utility-types'; +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { DataUsageConfigType } from '../config'; + +export interface DataUsageSetupDependencies { + features: FeaturesPluginSetup; +} + +/* eslint-disable @typescript-eslint/no-empty-interface*/ +export interface DataUsageStartDependencies {} + +export interface DataUsageServerSetup {} + +export interface DataUsageServerStart {} + +interface DataUsageApiRequestHandlerContext { + core: CoreRequestHandlerContext; +} + +export type DataUsageRequestHandlerContext = CustomRequestHandlerContext<{ + dataUsage: DataUsageApiRequestHandlerContext; +}>; + +export type DataUsageRouter = IRouter; + +export interface DataUsageContext { + logFactory: LoggerFactory; + serverConfig: DeepReadonly; +} diff --git a/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts b/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts new file mode 100644 index 0000000000000..a7f00a0e82a3d --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class CustomHttpRequestError extends Error { + constructor( + message: string, + public readonly statusCode: number = 500, + public readonly meta?: unknown + ) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } + } +} diff --git a/x-pack/plugins/data_usage/server/utils/index.ts b/x-pack/plugins/data_usage/server/utils/index.ts new file mode 100644 index 0000000000000..78b123f7c2bdf --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { isValidateTimeRange } from './validate_time_range'; +export { CustomHttpRequestError } from './custom_http_request_error'; diff --git a/x-pack/plugins/data_usage/server/utils/validate_time_range.ts b/x-pack/plugins/data_usage/server/utils/validate_time_range.ts new file mode 100644 index 0000000000000..8a311c991e9fa --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/validate_time_range.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export const isValidateTimeRange = (fromTimestamp: number, endTimestamp: number) => { + if ( + fromTimestamp < endTimestamp && + fromTimestamp > 0 && + endTimestamp > 0 && + fromTimestamp !== endTimestamp + ) { + return true; + } + return false; +}; From c94762f1e2bda3307a23c6e29d35406161673ac8 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Mon, 30 Sep 2024 15:41:51 +0200 Subject: [PATCH 33/53] data streams internal API --- .../data_usage/server/routes/index.tsx | 3 +- .../server/routes/internal/data_streams.ts | 35 +++++++++++++++ .../routes/internal/data_streams_handler.ts | 43 +++++++++++++++++++ .../server/routes/internal/index.tsx | 2 + .../server/types/rest_types/data_streams.ts | 18 ++++++++ .../server/types/rest_types/index.ts | 1 + 6 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/data_usage/server/routes/internal/data_streams.ts create mode 100644 x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts create mode 100644 x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts diff --git a/x-pack/plugins/data_usage/server/routes/index.tsx b/x-pack/plugins/data_usage/server/routes/index.tsx index b2d10b0ab53bc..b6b80c38864f3 100644 --- a/x-pack/plugins/data_usage/server/routes/index.tsx +++ b/x-pack/plugins/data_usage/server/routes/index.tsx @@ -6,11 +6,12 @@ */ import { DataUsageContext, DataUsageRouter } from '../types'; -import { registerUsageMetricsRoute } from './internal'; +import { registerDataStreamsRoute, registerUsageMetricsRoute } from './internal'; export const registerDataUsageRoutes = ( router: DataUsageRouter, dataUsageContext: DataUsageContext ) => { registerUsageMetricsRoute(router, dataUsageContext); + registerDataStreamsRoute(router, dataUsageContext); }; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts new file mode 100644 index 0000000000000..f3a401853f0ab --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataUsageContext, DataUsageRouter, DataStreamsResponseSchema } from '../../types'; + +import { getDataStreamsHandler } from './data_streams_handler'; + +export const registerDataStreamsRoute = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + if (dataUsageContext.serverConfig.enabled) { + router.versioned + .get({ + access: 'internal', + path: '/internal/api/data_usage/data_streams', + }) + .addVersion( + { + version: '1', + validate: { + request: {}, + response: { + 200: DataStreamsResponseSchema, + }, + }, + }, + getDataStreamsHandler(dataUsageContext) + ); + } +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts new file mode 100644 index 0000000000000..686edd0c4f4b7 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandler } from '@kbn/core/server'; +import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; + +import { errorHandler } from '../error_handler'; + +export const getDataStreamsHandler = ( + dataUsageContext: DataUsageContext +): RequestHandler => { + const logger = dataUsageContext.logFactory.get('dataStreamsRoute'); + + return async (context, _, response) => { + logger.debug(`Retrieving user data streams`); + + try { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + + const { data_streams: dataStreamsResponse } = await esClient.indices.dataStreamsStats({ + name: '*', + expand_wildcards: 'all', + }); + + const sorted = dataStreamsResponse + .sort((a, b) => b.store_size_bytes - a.store_size_bytes) + .map((dataStream) => ({ + name: dataStream.data_stream, + storageSizeBytes: dataStream.store_size_bytes, + })); + return response.ok({ + body: sorted, + }); + } catch (error) { + return errorHandler(logger, response, error); + } + }; +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/index.tsx b/x-pack/plugins/data_usage/server/routes/internal/index.tsx index d623dd8dd0ef7..e8d874bb7e6af 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/index.tsx +++ b/x-pack/plugins/data_usage/server/routes/internal/index.tsx @@ -4,4 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + export { registerUsageMetricsRoute } from './usage_metrics'; +export { registerDataStreamsRoute } from './data_streams'; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts b/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts new file mode 100644 index 0000000000000..b1c02bb40854d --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +export const DataStreamsResponseSchema = { + body: () => + schema.arrayOf( + schema.object({ + name: schema.string(), + storageSizeBytes: schema.number(), + }) + ), +}; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/index.ts b/x-pack/plugins/data_usage/server/types/rest_types/index.ts index 2a6dc5006d53e..64b5c640ebbb5 100644 --- a/x-pack/plugins/data_usage/server/types/rest_types/index.ts +++ b/x-pack/plugins/data_usage/server/types/rest_types/index.ts @@ -6,3 +6,4 @@ */ export * from './usage_metrics'; +export * from './data_streams'; From 44ca8a9680c95fdee97fa9e2686ae52329f37a25 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 2 Oct 2024 09:25:58 +0200 Subject: [PATCH 34/53] public hook to fetch metrics data --- x-pack/plugins/data_usage/common/index.ts | 4 + .../rest_types/data_streams.ts | 0 .../types => common}/rest_types/index.ts | 0 .../rest_types/usage_metrics.test.ts | 0 .../rest_types/usage_metrics.ts | 5 +- .../public/hooks/use_get_usage_metrics.ts | 46 ++++ .../server/routes/internal/data_streams.ts | 6 +- .../server/routes/internal/usage_metrics.ts | 11 +- .../routes/internal/usage_metrics_handler.ts | 239 ++++++++++++------ .../plugins/data_usage/server/types/index.ts | 1 - 10 files changed, 227 insertions(+), 85 deletions(-) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/data_streams.ts (100%) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/index.ts (100%) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/usage_metrics.test.ts (100%) rename x-pack/plugins/data_usage/{server/types => common}/rest_types/usage_metrics.ts (95%) create mode 100644 x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts diff --git a/x-pack/plugins/data_usage/common/index.ts b/x-pack/plugins/data_usage/common/index.ts index 4b6f899b58d37..eb0787f53f344 100644 --- a/x-pack/plugins/data_usage/common/index.ts +++ b/x-pack/plugins/data_usage/common/index.ts @@ -11,3 +11,7 @@ export const PLUGIN_ID = 'data_usage'; export const PLUGIN_NAME = i18n.translate('xpack.dataUsage.name', { defaultMessage: 'Data Usage', }); + +export const DATA_USAGE_API_ROUTE_PREFIX = '/api/data_usage/'; +export const DATA_USAGE_METRICS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}metrics`; +export const DATA_USAGE_DATA_STREAMS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}data_streams`; diff --git a/x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts b/x-pack/plugins/data_usage/common/rest_types/data_streams.ts similarity index 100% rename from x-pack/plugins/data_usage/server/types/rest_types/data_streams.ts rename to x-pack/plugins/data_usage/common/rest_types/data_streams.ts diff --git a/x-pack/plugins/data_usage/server/types/rest_types/index.ts b/x-pack/plugins/data_usage/common/rest_types/index.ts similarity index 100% rename from x-pack/plugins/data_usage/server/types/rest_types/index.ts rename to x-pack/plugins/data_usage/common/rest_types/index.ts diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts similarity index 100% rename from x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.test.ts rename to x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts diff --git a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts similarity index 95% rename from x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts rename to x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 5fe81567d94dc..61dbf85e3516c 100644 --- a/x-pack/plugins/data_usage/server/types/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -19,6 +19,8 @@ const METRIC_TYPE_VALUES = [ 'search_rate', ] as const; +export type MetricTypes = (typeof METRIC_TYPE_VALUES)[number]; + // @ts-ignore const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); @@ -62,7 +64,6 @@ export const UsageMetricsRequestSchema = { schema.oneOf([ schema.arrayOf(schema.string(), { minSize: 1, - maxSize: 50, // TBD validate: (values) => { if (values.map((v) => v.trim()).some((v) => !v.length)) { return '[dataStreams] list can not contain empty values'; @@ -102,3 +103,5 @@ export const UsageMetricsResponseSchema = { ), }), }; + +export type UsageMetricsResponseSchemaBody = TypeOf; diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts new file mode 100644 index 0000000000000..046579e6ac004 --- /dev/null +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; +import type { IHttpFetchError } from '@kbn/core-http-browser'; +import { + UsageMetricsRequestSchemaQueryParams, + UsageMetricsResponseSchemaBody, +} from '../../common/rest_types'; +import { DATA_USAGE_METRICS_API_ROUTE } from '../../common'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; + +interface ErrorType { + statusCode: number; + message: string; +} + +export const useGetDataUsageMetrics = ( + query: UsageMetricsRequestSchemaQueryParams, + options: UseQueryOptions> = {} +): UseQueryResult> => { + const http = useKibanaContextForPlugin().services.http; + + return useQuery>({ + queryKey: ['get-data-usage-metrics', query], + ...options, + keepPreviousData: true, + queryFn: async () => { + return http.get(DATA_USAGE_METRICS_API_ROUTE, { + version: '2023-10-31', + query: { + from: query.from, + to: query.to, + metricTypes: query.metricTypes, + size: query.size, + dataStreams: query.dataStreams, + }, + }); + }, + }); +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts index f3a401853f0ab..0d71d93b55849 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts @@ -5,7 +5,9 @@ * 2.0. */ -import { DataUsageContext, DataUsageRouter, DataStreamsResponseSchema } from '../../types'; +import { DataStreamsResponseSchema } from '../../../common/rest_types'; +import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../../common'; +import { DataUsageContext, DataUsageRouter } from '../../types'; import { getDataStreamsHandler } from './data_streams_handler'; @@ -17,7 +19,7 @@ export const registerDataStreamsRoute = ( router.versioned .get({ access: 'internal', - path: '/internal/api/data_usage/data_streams', + path: DATA_USAGE_DATA_STREAMS_API_ROUTE, }) .addVersion( { diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts index 439b0d4561f12..5bf3008ef668a 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -5,12 +5,9 @@ * 2.0. */ -import { - DataUsageContext, - DataUsageRouter, - UsageMetricsRequestSchema, - UsageMetricsResponseSchema, -} from '../../types'; +import { UsageMetricsRequestSchema, UsageMetricsResponseSchema } from '../../../common/rest_types'; +import { DATA_USAGE_METRICS_API_ROUTE } from '../../../common'; +import { DataUsageContext, DataUsageRouter } from '../../types'; import { getUsageMetricsHandler } from './usage_metrics_handler'; @@ -22,7 +19,7 @@ export const registerUsageMetricsRoute = ( router.versioned .get({ access: 'internal', - path: '/internal/api/data_usage/metrics', + path: DATA_USAGE_METRICS_API_ROUTE, }) .addVersion( { diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 6369b13aadcc4..6ae9b0ffbe82d 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -7,14 +7,14 @@ import { RequestHandler } from '@kbn/core/server'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; -import { - DataUsageContext, - DataUsageRequestHandlerContext, - UsageMetricsRequestSchemaQueryParams, -} from '../../types'; +import { MetricTypes, UsageMetricsRequestSchemaQueryParams } from '../../../common/rest_types'; +import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; import { errorHandler } from '../error_handler'; +const formatStringParams = (value: T | T[]): T[] | MetricTypes[] => + typeof value === 'string' ? [value] : value; + export const getUsageMetricsHandler = ( dataUsageContext: DataUsageContext ): RequestHandler< @@ -57,76 +57,15 @@ export const getUsageMetricsHandler = ( }, }); } - // TODO: fetch data from autoOps using userDsNames - // mock data - const charts = [ - { - key: 'ingest_rate', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 13756849 }, - { x: 1726862130000, y: 14657904 }, - { x: 1726865730000, y: 12798561 }, - { x: 1726869330000, y: 13578213 }, - { x: 1726872930000, y: 14123495 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - ], - }, - { - streamName: 'data_stream_3', - data: [{ x: 1726858530000, y: 12576413 }], - }, - ], - }, - { - key: 'storage_retained', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - ], - }, - { - streamName: 'data_stream_3', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - ], - }, - ], - }, - ]; + const charts = await fetchMetricsFromAutoOps({ + from, + to, + size, + metricTypes: formatStringParams(metricTypes) as MetricTypes[], + dataStreams: formatStringParams(userDsNames), + }); + return response.ok({ body: { charts, @@ -137,3 +76,155 @@ export const getUsageMetricsHandler = ( } }; }; + +const fetchMetricsFromAutoOps = async ({ + from, + to, + size, + metricTypes, + dataStreams, +}: { + from: string; + to: string; + size?: number; + metricTypes: MetricTypes[]; + dataStreams: string[]; +}) => { + // TODO: fetch data from autoOps using userDsNames + + const charts = [ + { + key: 'ingest_rate', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 13756849 }, + { x: 1726862130000, y: 14657904 }, + { x: 1726865730000, y: 12798561 }, + { x: 1726869330000, y: 13578213 }, + { x: 1726872930000, y: 14123495 }, + { x: 1726876530000, y: 13876548 }, + { x: 1726880130000, y: 12894561 }, + { x: 1726883730000, y: 14478953 }, + { x: 1726887330000, y: 14678905 }, + { x: 1726890930000, y: 13976547 }, + { x: 1726894530000, y: 14568945 }, + { x: 1726898130000, y: 13789561 }, + { x: 1726901730000, y: 14478905 }, + { x: 1726905330000, y: 13956423 }, + { x: 1726908930000, y: 14598234 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + ], + }, + { + key: 'storage_retained', + series: [ + { + streamName: 'data_stream_1', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + { + streamName: 'data_stream_2', + data: [ + { x: 1726858530000, y: 12894623 }, + { x: 1726862130000, y: 14436905 }, + { x: 1726865730000, y: 13794805 }, + { x: 1726869330000, y: 14048532 }, + { x: 1726872930000, y: 14237495 }, + { x: 1726876530000, y: 13745689 }, + { x: 1726880130000, y: 13974562 }, + { x: 1726883730000, y: 14234653 }, + { x: 1726887330000, y: 14323479 }, + { x: 1726890930000, y: 14023945 }, + { x: 1726894530000, y: 14189673 }, + { x: 1726898130000, y: 14247895 }, + { x: 1726901730000, y: 14098324 }, + { x: 1726905330000, y: 14478905 }, + { x: 1726908930000, y: 14323894 }, + ], + }, + { + streamName: 'data_stream_3', + data: [ + { x: 1726858530000, y: 12576413 }, + { x: 1726862130000, y: 13956423 }, + { x: 1726865730000, y: 14568945 }, + { x: 1726869330000, y: 14234856 }, + { x: 1726872930000, y: 14368942 }, + { x: 1726876530000, y: 13897654 }, + { x: 1726880130000, y: 14456989 }, + { x: 1726883730000, y: 14568956 }, + { x: 1726887330000, y: 13987562 }, + { x: 1726890930000, y: 14567894 }, + { x: 1726894530000, y: 14246789 }, + { x: 1726898130000, y: 14567895 }, + { x: 1726901730000, y: 14457896 }, + { x: 1726905330000, y: 14567895 }, + { x: 1726908930000, y: 13989456 }, + ], + }, + // Repeat similar structure for more data streams... + ], + }, + ]; + + return charts; +}; diff --git a/x-pack/plugins/data_usage/server/types/index.ts b/x-pack/plugins/data_usage/server/types/index.ts index 96577c72a8116..6cc0ccaa93a6d 100644 --- a/x-pack/plugins/data_usage/server/types/index.ts +++ b/x-pack/plugins/data_usage/server/types/index.ts @@ -5,5 +5,4 @@ * 2.0. */ -export * from './rest_types'; export * from './types'; From e1e7d33757c49086cc941e08421e53f5b3eddd52 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Wed, 2 Oct 2024 18:22:32 +0200 Subject: [PATCH 35/53] plug metrics API with charts UI --- .../data_usage/common/query_client.tsx | 57 +++++++ .../public/app/components/charts.tsx | 6 +- .../data_usage/public/app/data_usage.tsx | 146 ++---------------- x-pack/plugins/data_usage/public/app/types.ts | 5 +- .../plugins/data_usage/public/application.tsx | 5 +- .../public/hooks/use_get_usage_metrics.ts | 2 +- 6 files changed, 77 insertions(+), 144 deletions(-) create mode 100644 x-pack/plugins/data_usage/common/query_client.tsx diff --git a/x-pack/plugins/data_usage/common/query_client.tsx b/x-pack/plugins/data_usage/common/query_client.tsx new file mode 100644 index 0000000000000..8a64ed7a51349 --- /dev/null +++ b/x-pack/plugins/data_usage/common/query_client.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PropsWithChildren } from 'react'; +import React, { memo, useMemo } from 'react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; + +type QueryClientOptionsProp = ConstructorParameters[0]; + +/** + * Default Query Client for Data Usage. + */ +export class DataUsageQueryClient extends QueryClient { + constructor(options: QueryClientOptionsProp = {}) { + const optionsWithDefaults: QueryClientOptionsProp = { + ...options, + defaultOptions: { + ...(options.defaultOptions ?? {}), + queries: { + refetchIntervalInBackground: false, + refetchOnWindowFocus: false, + refetchOnMount: true, + keepPreviousData: true, + ...(options?.defaultOptions?.queries ?? {}), + }, + }, + }; + super(optionsWithDefaults); + } +} + +/** + * The default Data Usage Query Client. Can be imported and used from outside of React hooks + * and still benefit from ReactQuery features (like caching, etc) + * + * @see https://tanstack.com/query/v4/docs/reference/QueryClient + */ +export const dataUsageQueryClient = new DataUsageQueryClient(); + +export type ReactQueryClientProviderProps = PropsWithChildren<{ + queryClient?: DataUsageQueryClient; +}>; + +export const DataUsageReactQueryClientProvider = memo( + ({ queryClient, children }) => { + const client = useMemo(() => { + return queryClient || dataUsageQueryClient; + }, [queryClient]); + return {children}; + } +); + +DataUsageReactQueryClientProvider.displayName = 'DataUsageReactQueryClientProvider'; diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 08f51f60f887c..41b63d7623e88 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -40,11 +40,11 @@ const formatBytes = (bytes: number) => { return numeral(bytes).format('0.0 b'); }; -export const chartKeyToTitleMap: Record = { - ingestedMax: i18n.translate('xpack.dataUsage.charts.ingestedMax', { +export const chartKeyToTitleMap: Record = { + ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { defaultMessage: 'Data Ingested', }), - retainedMax: i18n.translate('xpack.dataUsage.charts.retainedMax', { + storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { defaultMessage: 'Data Retained in Storage', }), }; diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 476caaa8930a7..4c42f1b6141b2 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -18,153 +18,25 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { Charts } from './components/charts'; import { DatePicker } from './components/date_picker'; -import { MetricsResponse } from './types'; import { useBreadcrumbs } from '../utils/use_breadcrumbs'; import { useKibanaContextForPlugin } from '../utils/use_kibana'; import { PLUGIN_NAME } from '../../common'; import { DatePickerProvider } from './hooks/use_date_picker'; - -const response: MetricsResponse = { - charts: [ - { - key: 'ingestedMax', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 13756849 }, - { x: 1726862130000, y: 14657904 }, - { x: 1726865730000, y: 12798561 }, - { x: 1726869330000, y: 13578213 }, - { x: 1726872930000, y: 14123495 }, - { x: 1726876530000, y: 13876548 }, - { x: 1726880130000, y: 12894561 }, - { x: 1726883730000, y: 14478953 }, - { x: 1726887330000, y: 14678905 }, - { x: 1726890930000, y: 13976547 }, - { x: 1726894530000, y: 14568945 }, - { x: 1726898130000, y: 13789561 }, - { x: 1726901730000, y: 14478905 }, - { x: 1726905330000, y: 13956423 }, - { x: 1726908930000, y: 14598234 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - { x: 1726869330000, y: 14048532 }, - { x: 1726872930000, y: 14237495 }, - { x: 1726876530000, y: 13745689 }, - { x: 1726880130000, y: 13974562 }, - { x: 1726883730000, y: 14234653 }, - { x: 1726887330000, y: 14323479 }, - { x: 1726890930000, y: 14023945 }, - { x: 1726894530000, y: 14189673 }, - { x: 1726898130000, y: 14247895 }, - { x: 1726901730000, y: 14098324 }, - { x: 1726905330000, y: 14478905 }, - { x: 1726908930000, y: 14323894 }, - ], - }, - { - streamName: 'data_stream_3', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, - ], - }, - ], - }, - { - key: 'retainedMax', - series: [ - { - streamName: 'data_stream_1', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, - ], - }, - { - streamName: 'data_stream_2', - data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - { x: 1726869330000, y: 14048532 }, - { x: 1726872930000, y: 14237495 }, - { x: 1726876530000, y: 13745689 }, - { x: 1726880130000, y: 13974562 }, - { x: 1726883730000, y: 14234653 }, - { x: 1726887330000, y: 14323479 }, - { x: 1726890930000, y: 14023945 }, - { x: 1726894530000, y: 14189673 }, - { x: 1726898130000, y: 14247895 }, - { x: 1726901730000, y: 14098324 }, - { x: 1726905330000, y: 14478905 }, - { x: 1726908930000, y: 14323894 }, - ], - }, - { - streamName: 'data_stream_3', - data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, - ], - }, - // Repeat similar structure for more data streams... - ], - }, - ], -}; +import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; export const DataUsage = () => { const { services: { chrome, appParams }, } = useKibanaContextForPlugin(); + const { data, isFetching, isError } = useGetDataUsageMetrics({ + metricTypes: ['storage_retained', 'ingest_rate'], + to: 1726908930000, + from: 1726858530000, + }); + + const isLoading = isFetching || !data; + useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); return ( diff --git a/x-pack/plugins/data_usage/public/app/types.ts b/x-pack/plugins/data_usage/public/app/types.ts index 279a7e0e863ef..74e2d285f232b 100644 --- a/x-pack/plugins/data_usage/public/app/types.ts +++ b/x-pack/plugins/data_usage/public/app/types.ts @@ -4,7 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { MetricKey } from '../../common/types'; + +import { MetricTypes } from '../../common/rest_types'; export interface DataPoint { x: number; @@ -16,7 +17,7 @@ export interface Series { data: DataPoint[]; } export interface Chart { - key: MetricKey; + key: MetricTypes; series: Series[]; } diff --git a/x-pack/plugins/data_usage/public/application.tsx b/x-pack/plugins/data_usage/public/application.tsx index a160a5742d36d..054aae397e5e1 100644 --- a/x-pack/plugins/data_usage/public/application.tsx +++ b/x-pack/plugins/data_usage/public/application.tsx @@ -17,6 +17,7 @@ import { useKibanaContextForPluginProvider } from './utils/use_kibana'; import { DataUsageStartDependencies, DataUsagePublicStart } from './types'; import { PLUGIN_ID } from '../common'; import { DataUsage } from './app/data_usage'; +import { DataUsageReactQueryClientProvider } from '../common/query_client'; export const renderApp = ( core: CoreStart, @@ -77,7 +78,9 @@ const App = ({ core, plugins, pluginStart, params }: AppProps) => { return ( - + + + ); diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts index 046579e6ac004..91cee020f243e 100644 --- a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -32,7 +32,7 @@ export const useGetDataUsageMetrics = ( keepPreviousData: true, queryFn: async () => { return http.get(DATA_USAGE_METRICS_API_ROUTE, { - version: '2023-10-31', + version: '1', query: { from: query.from, to: query.to, From b2561c6a38eba9797d0bb365e26e5353fe49d1f3 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Fri, 4 Oct 2024 09:17:14 +0200 Subject: [PATCH 36/53] update date picker to work with API and update URL --- .../common/rest_types/usage_metrics.test.ts | 22 --- .../common/rest_types/usage_metrics.ts | 14 +- x-pack/plugins/data_usage/kibana.jsonc | 20 ++- .../public/app/components/charts.tsx | 6 +- .../public/app/components/date_picker.tsx | 118 +++++++++----- .../data_usage/public/app/data_usage.tsx | 71 +++++++-- .../app/hooks/use_charts_url_params.tsx | 146 ++++++++++++++++++ .../public/app/hooks/use_date_picker.tsx | 111 ++++++++++--- .../data_usage/public/hooks/use_url_params.ts | 30 ++++ x-pack/plugins/data_usage/tsconfig.json | 1 + 10 files changed, 430 insertions(+), 109 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx create mode 100644 x-pack/plugins/data_usage/public/hooks/use_url_params.ts diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts index de46215a70d85..1e04c84c113d1 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts @@ -11,8 +11,6 @@ describe('usage_metrics schemas', () => { it('should accept valid request query', () => { expect(() => UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: new Date().toISOString(), metricTypes: ['storage_retained'], }) ).not.toThrow(); @@ -151,24 +149,4 @@ describe('usage_metrics schemas', () => { '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' ); }); - - it('should error if `from` is not valid', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: 'bar', - to: new Date().toISOString(), - metricTypes: ['storage_retained'], - }) - ).toThrow('[from]: Invalid date'); - }); - - it('should error if `to` is not valid', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: 'foo', - metricTypes: ['storage_retained'], - }) - ).toThrow('[to]: Invalid date'); - }); }); diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 61dbf85e3516c..0c7971686ea43 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -21,9 +21,15 @@ const METRIC_TYPE_VALUES = [ export type MetricTypes = (typeof METRIC_TYPE_VALUES)[number]; +// type guard for MetricTypes +export const isMetricType = (type: string): type is MetricTypes => + METRIC_TYPE_VALUES.includes(type as MetricTypes); + // @ts-ignore const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); +const DateSchema = schema.maybe(schema.string()); + const metricTypesSchema = { // @ts-expect-error TS2769: No overload matches this call schema: schema.oneOf(METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType))), @@ -32,12 +38,8 @@ const metricTypesSchema = { export const UsageMetricsRequestSchema = { query: schema.object({ - from: schema.string({ - validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), - }), - to: schema.string({ - validate: (v) => (new Date(v).toString() === 'Invalid Date' ? 'Invalid date' : undefined), - }), + from: DateSchema, + to: DateSchema, size: schema.maybe(schema.number()), // should be same as dataStreams.length metricTypes: schema.oneOf([ schema.arrayOf(schema.string(), { diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 9b0f2d193925e..ffd8833351267 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -1,16 +1,28 @@ { "type": "plugin", "id": "@kbn/data-usage-plugin", - "owner": ["@elastic/obs-ai-assistant", "@elastic/security-solution"], + "owner": [ + "@elastic/obs-ai-assistant", + "@elastic/security-solution" + ], "plugin": { "id": "dataUsage", "server": true, "browser": true, - "configPath": ["xpack", "dataUsage"], - "requiredPlugins": ["home", "management", "features", "share"], + "configPath": [ + "xpack", + "dataUsage" + ], + "requiredPlugins": [ + "home", + "management", + "features", + "share" + ], "optionalPlugins": [], "requiredBundles": [ "kibanaReact", - ], + "data" + ] } } diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 41b63d7623e88..48e04c7906256 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -31,8 +31,8 @@ import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { MetricsResponse } from '../types'; -import { MetricKey } from '../../../common/types'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +import { MetricTypes } from '../../../common/rest_types'; interface ChartsProps { data: MetricsResponse; } @@ -40,7 +40,9 @@ const formatBytes = (bytes: number) => { return numeral(bytes).format('0.0 b'); }; -export const chartKeyToTitleMap: Record = { +// TODO: Remove this when we have a title for each metric type +type ChartKey = Extract; +export const chartKeyToTitleMap: Record = { ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { defaultMessage: 'Data Ingested', }), diff --git a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx index f95af6d6bfc43..ca29acf8c96a6 100644 --- a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx +++ b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx @@ -5,48 +5,84 @@ * 2.0. */ -import React from 'react'; -import { EuiSuperDatePicker } from '@elastic/eui'; -import { useDatePickerContext } from '../hooks/use_date_picker'; +import React, { memo, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiSuperDatePicker, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/react'; +import type { IUnifiedSearchPluginServices } from '@kbn/unified-search-plugin/public'; +import type { EuiSuperDatePickerRecentRange } from '@elastic/eui'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import type { + DurationRange, + OnRefreshChangeProps, +} from '@elastic/eui/src/components/date_picker/types'; +import { UI_SETTINGS } from '@kbn/data-plugin/common'; -export const DatePicker: React.FC = () => { - const { - startDate, - endDate, - isAutoRefreshActive, - refreshInterval, - setStartDate, - setEndDate, - setIsAutoRefreshActive, - setRefreshInterval, - } = useDatePickerContext(); - - const onTimeChange = ({ start, end }: { start: string; end: string }) => { - setStartDate(start); - setEndDate(end); - // Trigger API call or data refresh here +export interface DateRangePickerValues { + autoRefreshOptions: { + enabled: boolean; + duration: number; }; + startDate: string; + endDate: string; + recentlyUsedDateRanges: EuiSuperDatePickerRecentRange[]; +} - const onRefreshChange = ({ - isPaused, - refreshInterval: onRefreshRefreshInterval, - }: { - isPaused: boolean; - refreshInterval: number; - }) => { - setIsAutoRefreshActive(!isPaused); - setRefreshInterval(onRefreshRefreshInterval); - }; +interface UsageMetricsDateRangePickerProps { + dateRangePickerState: DateRangePickerValues; + isDataLoading: boolean; + onRefresh: () => void; + onRefreshChange: (evt: OnRefreshChangeProps) => void; + onTimeChange: ({ start, end }: DurationRange) => void; +} + +export const UsageMetricsDateRangePicker = memo( + ({ dateRangePickerState, isDataLoading, onRefresh, onRefreshChange, onTimeChange }) => { + const { euiTheme } = useEuiTheme(); + const kibana = useKibana(); + const { uiSettings } = kibana.services; + const [commonlyUsedRanges] = useState(() => { + return ( + uiSettings + ?.get(UI_SETTINGS.TIMEPICKER_QUICK_RANGES) + ?.map(({ from, to, display }: { from: string; to: string; display: string }) => { + return { + start: from, + end: to, + label: display, + }; + }) ?? [] + ); + }); + + return ( +
+ + + + + +
+ ); + } +); - return ( - - ); -}; +UsageMetricsDateRangePicker.displayName = 'UsageMetricsDateRangePicker'; diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 4c42f1b6141b2..77104eeb1d5c3 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -5,42 +5,81 @@ * 2.0. */ -import React from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { EuiTitle, EuiSpacer, EuiFlexGroup, EuiFlexItem, - EuiText, + EuiLoadingElastic, EuiPageSection, + EuiText, } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { UsageMetricsRequestSchemaQueryParams } from '../../common/rest_types'; import { Charts } from './components/charts'; -import { DatePicker } from './components/date_picker'; +import { UsageMetricsDateRangePicker } from './components/date_picker'; import { useBreadcrumbs } from '../utils/use_breadcrumbs'; import { useKibanaContextForPlugin } from '../utils/use_kibana'; import { PLUGIN_NAME } from '../../common'; -import { DatePickerProvider } from './hooks/use_date_picker'; import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; +import { useDateRangePicker } from './hooks/use_date_picker'; +import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params'; export const DataUsage = () => { const { services: { chrome, appParams }, } = useKibanaContextForPlugin(); - const { data, isFetching, isError } = useGetDataUsageMetrics({ + const { metricTypes: metricTypesFromUrl, dataStreams: dataStreamsFromUrl } = + useDataUsageMetricsUrlParams(); + + const [queryParams, setQueryParams] = useState({ metricTypes: ['storage_retained', 'ingest_rate'], - to: 1726908930000, - from: 1726858530000, + dataStreams: [], }); - const isLoading = isFetching || !data; + useEffect(() => { + setQueryParams((prevState) => ({ + ...prevState, + metricTypes: metricTypesFromUrl?.length ? metricTypesFromUrl : prevState.metricTypes, + dataStreams: dataStreamsFromUrl?.length ? dataStreamsFromUrl : prevState.dataStreams, + })); + }, [metricTypesFromUrl, dataStreamsFromUrl]); + + const { dateRangePickerState, onRefreshChange, onTimeChange } = useDateRangePicker(); + + const { + error, + data, + isFetching, + isFetched, + refetch: refetchDataUsageMetrics, + } = useGetDataUsageMetrics( + { + ...queryParams, + from: dateRangePickerState.startDate, + to: dateRangePickerState.endDate, + }, + { + retry: false, + } + ); + + const onRefresh = useCallback(() => { + refetchDataUsageMetrics(); + }, [refetchDataUsageMetrics]); useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); + // TODO: show a toast? + if (!isFetching && error?.body) { + return
{error.body.message}
; + } + return ( - + <>

{i18n.translate('xpack.dataUsage.pageTitle', { @@ -60,12 +99,18 @@ export const DataUsage = () => { - + - + {isFetched && data ? : } - + ); }; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx new file mode 100644 index 0000000000000..9b31e10b9e18e --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useHistory, useLocation } from 'react-router-dom'; +import { MetricTypes, isMetricType } from '../../../common/rest_types'; +import { useUrlParams } from '../../hooks/use_url_params'; + +interface UrlParamsDataUsageMetricsFilters { + metricTypes: string; + dataStreams: string; + startDate: string; + endDate: string; +} + +interface DataUsageMetricsFiltersFromUrlParams { + metricTypes?: MetricTypes[]; + dataStreams?: string[]; + startDate?: string; + endDate?: string; + setUrlDataStreamsFilter: (dataStreams: UrlParamsDataUsageMetricsFilters['dataStreams']) => void; + setUrlDateRangeFilter: ({ startDate, endDate }: { startDate: string; endDate: string }) => void; + setUrlMetricTypesFilter: (metricTypes: UrlParamsDataUsageMetricsFilters['metricTypes']) => void; +} + +type FiltersFromUrl = Pick< + DataUsageMetricsFiltersFromUrlParams, + 'metricTypes' | 'dataStreams' | 'startDate' | 'endDate' +>; + +export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({ + autoRefreshOptions: { + enabled: false, + duration: 10000, + }, + startDate: 'now-24h/h', + endDate: 'now', + recentlyUsedDateRanges: [], +}); + +export const getDataUsageMetricsFiltersFromUrlParams = ( + urlParams: Partial +): FiltersFromUrl => { + const dataUsageMetricsFilters: FiltersFromUrl = { + metricTypes: [], + dataStreams: [], + startDate: DEFAULT_DATE_RANGE_OPTIONS.startDate, + endDate: DEFAULT_DATE_RANGE_OPTIONS.endDate, + }; + + const urlMetricTypes = urlParams.metricTypes + ? (String(urlParams.metricTypes).split(',') as MetricTypes[]).reduce( + (acc, curr) => { + if (isMetricType(curr)) { + acc.push(curr); + } + return acc.sort(); + }, + [] + ) + : []; + + const urlDataStreams = urlParams.dataStreams + ? String(urlParams.dataStreams).split(',').sort() + : []; + + dataUsageMetricsFilters.metricTypes = urlMetricTypes.length ? urlMetricTypes : undefined; + dataUsageMetricsFilters.dataStreams = urlDataStreams.length ? urlDataStreams : undefined; + dataUsageMetricsFilters.startDate = urlParams.startDate ? String(urlParams.startDate) : undefined; + dataUsageMetricsFilters.endDate = urlParams.endDate ? String(urlParams.endDate) : undefined; + + return dataUsageMetricsFilters; +}; + +export const useDataUsageMetricsUrlParams = (): DataUsageMetricsFiltersFromUrlParams => { + const location = useLocation(); + const history = useHistory(); + const { urlParams, toUrlParams } = useUrlParams(); + + const getUrlDataUsageMetricsFilters: FiltersFromUrl = useMemo( + () => getDataUsageMetricsFiltersFromUrlParams(urlParams), + [urlParams] + ); + const [dataUsageMetricsFilters, setDataUsageMetricsFilters] = useState( + getUrlDataUsageMetricsFilters + ); + + const setUrlMetricTypesFilter = useCallback( + (metricTypes: string) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + hosts: metricTypes.length ? metricTypes : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + const setUrlDataStreamsFilter = useCallback( + (dataStreams: string) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + dataStreams: dataStreams.length ? dataStreams : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + const setUrlDateRangeFilter = useCallback( + ({ startDate, endDate }: { startDate: string; endDate: string }) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + startDate: startDate.length ? startDate : undefined, + endDate: endDate.length ? endDate : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + useEffect(() => { + setDataUsageMetricsFilters((prevState) => { + return { + ...prevState, + ...getDataUsageMetricsFiltersFromUrlParams(urlParams), + }; + }); + }, [setDataUsageMetricsFilters, urlParams]); + + return { + ...dataUsageMetricsFilters, + setUrlDataStreamsFilter, + setUrlDateRangeFilter, + setUrlMetricTypesFilter, + }; +}; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx index cd8ac35c98839..b5407ae9e46d5 100644 --- a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx +++ b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx @@ -5,25 +5,94 @@ * 2.0. */ -import { useState } from 'react'; -import createContainer from 'constate'; - -const useDatePicker = () => { - const [startDate, setStartDate] = useState('now-24h'); - const [endDate, setEndDate] = useState('now'); - const [isAutoRefreshActive, setIsAutoRefreshActive] = useState(false); - const [refreshInterval, setRefreshInterval] = useState(0); - - return { - startDate, - setStartDate, - endDate, - setEndDate, - isAutoRefreshActive, - setIsAutoRefreshActive, - refreshInterval, - setRefreshInterval, - }; -}; +import { useCallback, useState } from 'react'; +import type { + DurationRange, + OnRefreshChangeProps, +} from '@elastic/eui/src/components/date_picker/types'; +import { useDataUsageMetricsUrlParams } from './use_charts_url_params'; +import { DateRangePickerValues } from '../components/date_picker'; + +export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({ + autoRefreshOptions: { + enabled: false, + duration: 10000, + }, + startDate: 'now-24h/h', + endDate: 'now', + recentlyUsedDateRanges: [], +}); + +export const useDateRangePicker = () => { + const { + setUrlDateRangeFilter, + startDate: startDateFromUrl, + endDate: endDateFromUrl, + } = useDataUsageMetricsUrlParams(); + const [dateRangePickerState, setDateRangePickerState] = useState({ + ...DEFAULT_DATE_RANGE_OPTIONS, + startDate: startDateFromUrl ?? DEFAULT_DATE_RANGE_OPTIONS.startDate, + endDate: endDateFromUrl ?? DEFAULT_DATE_RANGE_OPTIONS.endDate, + }); + + const updateUsageMetricsDateRanges = useCallback( + ({ start, end }: DurationRange) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + startDate: start, + endDate: end, + })); + }, + [setDateRangePickerState] + ); + + const updateUsageMetricsRecentlyUsedDateRanges = useCallback( + (recentlyUsedDateRanges: DateRangePickerValues['recentlyUsedDateRanges']) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + recentlyUsedDateRanges, + })); + }, + [setDateRangePickerState] + ); -export const [DatePickerProvider, useDatePickerContext] = createContainer(useDatePicker); + // handle refresh timer update + const onRefreshChange = useCallback( + (evt: OnRefreshChangeProps) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + autoRefreshOptions: { enabled: !evt.isPaused, duration: evt.refreshInterval }, + })); + }, + [setDateRangePickerState] + ); + + // handle manual time change on date picker + const onTimeChange = useCallback( + ({ start: newStart, end: newEnd }: DurationRange) => { + // update date ranges + updateUsageMetricsDateRanges({ start: newStart, end: newEnd }); + + // update recently used date ranges + const newRecentlyUsedDateRanges = [ + { start: newStart, end: newEnd }, + ...dateRangePickerState.recentlyUsedDateRanges + .filter( + (recentlyUsedRange: DurationRange) => + !(recentlyUsedRange.start === newStart && recentlyUsedRange.end === newEnd) + ) + .slice(0, 9), + ]; + updateUsageMetricsRecentlyUsedDateRanges(newRecentlyUsedDateRanges); + setUrlDateRangeFilter({ startDate: newStart, endDate: newEnd }); + }, + [ + dateRangePickerState.recentlyUsedDateRanges, + setUrlDateRangeFilter, + updateUsageMetricsDateRanges, + updateUsageMetricsRecentlyUsedDateRanges, + ] + ); + + return { dateRangePickerState, onRefreshChange, onTimeChange }; +}; diff --git a/x-pack/plugins/data_usage/public/hooks/use_url_params.ts b/x-pack/plugins/data_usage/public/hooks/use_url_params.ts new file mode 100644 index 0000000000000..865b71781df63 --- /dev/null +++ b/x-pack/plugins/data_usage/public/hooks/use_url_params.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import { parse, stringify } from 'query-string'; +import { useLocation } from 'react-router-dom'; + +/** + * Parses `search` params and returns an object with them along with a `toUrlParams` function + * that allows being able to retrieve a stringified version of an object (default is the + * `urlParams` that was parsed) for use in the url. + * Object will be recreated every time `search` changes. + */ +export function useUrlParams>(): { + urlParams: T; + toUrlParams: (params?: T) => string; +} { + const { search } = useLocation(); + return useMemo(() => { + const urlParams = parse(search) as unknown as T; + return { + urlParams, + toUrlParams: (params: T = urlParams) => stringify(params as unknown as object), + }; + }, [search]); +} diff --git a/x-pack/plugins/data_usage/tsconfig.json b/x-pack/plugins/data_usage/tsconfig.json index ebc023568cf88..9087062db0b06 100644 --- a/x-pack/plugins/data_usage/tsconfig.json +++ b/x-pack/plugins/data_usage/tsconfig.json @@ -13,6 +13,7 @@ "kbn_references": [ "@kbn/core", "@kbn/i18n", + "@kbn/data-plugin", "@kbn/kibana-react-plugin", "@kbn/management-plugin", "@kbn/react-kibana-context-render", From 8ee050ade377a17c879686781365a272488829ba Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 12:05:22 -0400 Subject: [PATCH 37/53] add kibana index link and separate out dataset quality link component --- .../public/app/components/charts.tsx | 47 +++++++------------ .../app/components/dataset_quality_link.tsx | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 48e04c7906256..9d198a348dc80 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -29,10 +29,10 @@ import { } from '@elastic/charts'; import numeral from '@elastic/numeral'; import { i18n } from '@kbn/i18n'; -import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { MetricsResponse } from '../types'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; import { MetricTypes } from '../../../common/rest_types'; +import { DatasetQualityLink } from './dataset_quality_link'; interface ChartsProps { data: MetricsResponse; } @@ -50,13 +50,7 @@ export const chartKeyToTitleMap: Record = { defaultMessage: 'Data Retained in Storage', }), }; -function getDatasetFromDataStream(dataStreamName: string): string | null { - const parts = dataStreamName.split('-'); - if (parts.length !== 3) { - return null; - } - return parts[1]; -} + export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); const theme = useEuiTheme(); @@ -69,20 +63,16 @@ export const Charts: React.FC = ({ data }) => { }, } = useKibanaContextForPlugin(); const hasDataSetQualityFeature = capabilities.data_quality; - const onClickDataQuality = async ({ dataStreamName }: { dataStreamName: string }) => { - const dataQualityLocator = locators.get(DATA_QUALITY_LOCATOR_ID); - const locatorParams: DataQualityLocatorParams = { - filters: { - // TODO: get time range from our page state - timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, - }, - }; - const dataset = getDatasetFromDataStream(dataStreamName); - if (locatorParams?.filters && dataset) { - locatorParams.filters.query = dataset; - } + const hasIndexManagementFeature = capabilities.index_management; + + const onClickIndexManagement = async ({ dataStreamName }: { dataStreamName: string }) => { + // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 + const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); if (dataQualityLocator) { - await dataQualityLocator.navigate(locatorParams); + await dataQualityLocator.navigate({ + sectionId: 'data', + appId: `index_management/data_streams/${dataStreamName}`, + }); } }; @@ -136,21 +126,20 @@ export const Charts: React.FC = ({ data }) => { label="Copy data stream name" onClick={() => undefined} /> - undefined} - /> - {hasDataSetQualityFeature && ( + {hasIndexManagementFeature && ( - onClickDataQuality({ + onClickIndexManagement({ dataStreamName: label, }) } /> )} + {hasDataSetQualityFeature && ( + + )} diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx new file mode 100644 index 0000000000000..f095563a65a6e --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiListGroupItem } from '@elastic/eui'; + +import React from 'react'; +import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; + +function getDatasetFromDataStream(dataStreamName: string): string | null { + const parts = dataStreamName.split('-'); + if (parts.length !== 3) { + return null; + } + return parts[1]; +} + +export const DatasetQualityLink = React.memo(({ dataStreamName }: { dataStreamName: string }) => { + const { + services: { + share: { url }, + }, + } = useKibanaContextForPlugin(); + + const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); + const onClickDataQuality = async () => { + const locatorParams: DataQualityLocatorParams = { + filters: { + // TODO: get time range from our page state + timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + }, + }; + const dataset = getDatasetFromDataStream(dataStreamName); + if (locatorParams?.filters && dataset) { + locatorParams.filters.query = dataset; + } + if (locator) { + await locator.navigate(locatorParams); + } + }; + + return onClickDataQuality()} />; +}); From b68749c7af3992416628e58f87bba5c76cb6b1c8 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 12:43:59 -0400 Subject: [PATCH 38/53] app copy function and spacer --- .../plugins/data_usage/public/app/components/charts.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 9d198a348dc80..956cfec193636 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -13,6 +13,7 @@ import { EuiListGroupItem, EuiPanel, EuiPopover, + EuiSpacer, EuiTitle, useEuiTheme, } from '@elastic/eui'; @@ -122,10 +123,13 @@ export const Charts: React.FC = ({ data }) => { > undefined} + onClick={() => { + navigator.clipboard.writeText(label); + }} /> + + {hasIndexManagementFeature && ( Date: Fri, 4 Oct 2024 13:29:13 -0400 Subject: [PATCH 39/53] create LegendAction component --- .../public/app/components/charts.tsx | 100 ++++-------------- .../public/app/components/legend_action.tsx | 95 +++++++++++++++++ 2 files changed, 113 insertions(+), 82 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/components/legend_action.tsx diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 956cfec193636..3b9bc54c89003 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -4,19 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useState } from 'react'; -import { - EuiButtonIcon, - EuiFlexGroup, - EuiFlexItem, - EuiListGroup, - EuiListGroupItem, - EuiPanel, - EuiPopover, - EuiSpacer, - EuiTitle, - useEuiTheme, -} from '@elastic/eui'; +import React, { useCallback, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; import { Chart, Axis, @@ -24,7 +13,6 @@ import { Settings, ScaleType, niceTimeFormatter, - LegendActionProps, DARK_THEME, LIGHT_THEME, } from '@elastic/charts'; @@ -33,7 +21,7 @@ import { i18n } from '@kbn/i18n'; import { MetricsResponse } from '../types'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; import { MetricTypes } from '../../../common/rest_types'; -import { DatasetQualityLink } from './dataset_quality_link'; +import { LegendAction } from './legend_action'; interface ChartsProps { data: MetricsResponse; } @@ -57,29 +45,15 @@ export const Charts: React.FC = ({ data }) => { const theme = useEuiTheme(); const { services: { - share: { - url: { locators }, - }, application: { capabilities }, }, } = useKibanaContextForPlugin(); - const hasDataSetQualityFeature = capabilities.data_quality; - const hasIndexManagementFeature = capabilities.index_management; - - const onClickIndexManagement = async ({ dataStreamName }: { dataStreamName: string }) => { - // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 - const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); - if (dataQualityLocator) { - await dataQualityLocator.navigate({ - sectionId: 'data', - appId: `index_management/data_streams/${dataStreamName}`, - }); - } - }; + const hasDataSetQualityFeature = !!capabilities?.data_quality; + const hasIndexManagementFeature = !!capabilities?.index_management; - const togglePopover = (streamName: string) => { - setPopoverOpen(popoverOpen === streamName ? null : streamName); - }; + const togglePopover = useCallback((streamName: string | null) => { + setPopoverOpen((prev) => (prev === streamName ? null : streamName)); + }, []); return ( {data.charts.map((chart, idx) => { @@ -101,54 +75,16 @@ export const Charts: React.FC = ({ data }) => { showLegend={true} legendPosition="right" xDomain={{ min: minTimestamp, max: maxTimestamp }} - legendAction={({ label }: LegendActionProps) => { - const uniqueStreamName = `${idx}-${label}`; - return ( - - - - togglePopover(uniqueStreamName)} - /> - - - } - isOpen={popoverOpen === uniqueStreamName} - closePopover={() => setPopoverOpen(null)} - anchorPosition="downRight" - > - - { - navigator.clipboard.writeText(label); - }} - /> - - - {hasIndexManagementFeature && ( - - onClickIndexManagement({ - dataStreamName: label, - }) - } - /> - )} - {hasDataSetQualityFeature && ( - - )} - - - - ); - }} + legendAction={({ label }) => ( + + )} /> {chart.series.map((stream, streamIdx) => ( void; + hasIndexManagementFeature: boolean; + hasDataSetQualityFeature: boolean; + label: string; +} + +export const LegendAction: React.FC = React.memo( + ({ + label, + idx, + popoverOpen, + togglePopover, + hasIndexManagementFeature, + hasDataSetQualityFeature, + }) => { + const uniqueStreamName = `${idx}-${label}`; + const { + services: { + share: { + url: { locators }, + }, + }, + } = useKibanaContextForPlugin(); + + const onClickIndexManagement = useCallback(async () => { + // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 + const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); + if (dataQualityLocator) { + await dataQualityLocator.navigate({ + sectionId: 'data', + appId: `index_management/data_streams/${label}`, + }); + } + togglePopover(null); // Close the popover after action + }, [label, locators, togglePopover]); + + const onCopyDataStreamName = useCallback(() => { + navigator.clipboard.writeText(label); + togglePopover(null); // Close popover after copying + }, [label, togglePopover]); + + return ( + + + + togglePopover(uniqueStreamName)} + /> + + + } + isOpen={popoverOpen === uniqueStreamName} + closePopover={() => togglePopover(null)} + anchorPosition="downRight" + > + + + + + {hasIndexManagementFeature && ( + + )} + {hasDataSetQualityFeature && } + + + + ); + } +); From bf72cda0b441fc3c93283ff3adf3151065211c59 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 13:44:39 -0400 Subject: [PATCH 40/53] clean up DatasetQualityLink component --- .../app/components/dataset_quality_link.tsx | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx index f095563a65a6e..d71093be9bb4a 100644 --- a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -5,12 +5,44 @@ * 2.0. */ -import { EuiListGroupItem } from '@elastic/eui'; - import React from 'react'; +import { EuiListGroupItem } from '@elastic/eui'; import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +interface DatasetQualityLinkProps { + dataStreamName: string; +} + +export const DatasetQualityLink: React.FC = React.memo( + ({ dataStreamName }) => { + const { + services: { + share: { url }, + }, + } = useKibanaContextForPlugin(); + + const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); + + const onClickDataQuality = async () => { + const locatorParams: DataQualityLocatorParams = { + filters: { + // TODO: get time range from our page state + timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + }, + }; + const dataset = getDatasetFromDataStream(dataStreamName); + if (locatorParams?.filters && dataset) { + locatorParams.filters.query = dataset; + } + if (locator) { + await locator.navigate(locatorParams); + } + }; + return ; + } +); + function getDatasetFromDataStream(dataStreamName: string): string | null { const parts = dataStreamName.split('-'); if (parts.length !== 3) { @@ -18,30 +50,3 @@ function getDatasetFromDataStream(dataStreamName: string): string | null { } return parts[1]; } - -export const DatasetQualityLink = React.memo(({ dataStreamName }: { dataStreamName: string }) => { - const { - services: { - share: { url }, - }, - } = useKibanaContextForPlugin(); - - const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); - const onClickDataQuality = async () => { - const locatorParams: DataQualityLocatorParams = { - filters: { - // TODO: get time range from our page state - timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, - }, - }; - const dataset = getDatasetFromDataStream(dataStreamName); - if (locatorParams?.filters && dataset) { - locatorParams.filters.query = dataset; - } - if (locator) { - await locator.navigate(locatorParams); - } - }; - - return onClickDataQuality()} />; -}); From cc1bf4cc6158f72a2ec1da5405d57c3607e414f0 Mon Sep 17 00:00:00 2001 From: neptunian Date: Fri, 4 Oct 2024 15:02:39 -0400 Subject: [PATCH 41/53] separate out components --- .../public/app/components/chart_panel.tsx | 121 ++++++++++++++++++ .../public/app/components/charts.tsx | 114 ++--------------- .../public/app/components/legend_action.tsx | 14 +- 3 files changed, 137 insertions(+), 112 deletions(-) create mode 100644 x-pack/plugins/data_usage/public/app/components/chart_panel.tsx diff --git a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx new file mode 100644 index 0000000000000..e17ad2692c63a --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo } from 'react'; +import numeral from '@elastic/numeral'; +import { EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; +import { + Chart, + Axis, + BarSeries, + Settings, + ScaleType, + niceTimeFormatter, + DARK_THEME, + LIGHT_THEME, +} from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import { LegendAction } from './legend_action'; +import { MetricTypes } from '../../../common/rest_types'; +import { Chart as ChartData } from '../types'; + +// TODO: Remove this when we have a title for each metric type +type ChartKey = Extract; +export const chartKeyToTitleMap: Record = { + ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { + defaultMessage: 'Data Ingested', + }), + storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { + defaultMessage: 'Data Retained in Storage', + }), +}; + +interface ChartPanelProps { + data: ChartData; + idx: number; + popoverOpen: string | null; + togglePopover: (streamName: string | null) => void; +} + +export const ChartPanel: React.FC = ({ + data, + idx, + popoverOpen, + togglePopover, +}) => { + const theme = useEuiTheme(); + + const chartTimestamps = data.series.flatMap((series) => series.data.map((d) => d.x)); + + const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)]; + + const tickFormat = useMemo( + () => niceTimeFormatter([minTimestamp, maxTimestamp]), + [minTimestamp, maxTimestamp] + ); + + const renderLegendAction = useCallback( + ({ label }: { label: string }) => { + return ( + + ); + }, + [idx, popoverOpen, togglePopover] + ); + return ( + + + +
{chartKeyToTitleMap[data.key as ChartKey] || data.key}
+ + + + {data.series.map((stream, streamIdx) => ( + [point.x, point.y])} + xScaleType={ScaleType.Time} + yScaleType={ScaleType.Linear} + xAccessor={0} + yAccessors={[1]} + stackAccessors={[0]} + /> + ))} + + + + formatBytes(d)} + /> + + + + ); +}; +const formatBytes = (bytes: number) => { + return numeral(bytes).format('0.0 b'); +}; diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index 3b9bc54c89003..ccf753d579f20 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -5,120 +5,30 @@ * 2.0. */ import React, { useCallback, useState } from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; -import { - Chart, - Axis, - BarSeries, - Settings, - ScaleType, - niceTimeFormatter, - DARK_THEME, - LIGHT_THEME, -} from '@elastic/charts'; -import numeral from '@elastic/numeral'; -import { i18n } from '@kbn/i18n'; +import { EuiFlexGroup } from '@elastic/eui'; import { MetricsResponse } from '../types'; -import { useKibanaContextForPlugin } from '../../utils/use_kibana'; -import { MetricTypes } from '../../../common/rest_types'; -import { LegendAction } from './legend_action'; +import { ChartPanel } from './chart_panel'; interface ChartsProps { data: MetricsResponse; } -const formatBytes = (bytes: number) => { - return numeral(bytes).format('0.0 b'); -}; - -// TODO: Remove this when we have a title for each metric type -type ChartKey = Extract; -export const chartKeyToTitleMap: Record = { - ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { - defaultMessage: 'Data Ingested', - }), - storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { - defaultMessage: 'Data Retained in Storage', - }), -}; export const Charts: React.FC = ({ data }) => { const [popoverOpen, setPopoverOpen] = useState(null); - const theme = useEuiTheme(); - const { - services: { - application: { capabilities }, - }, - } = useKibanaContextForPlugin(); - const hasDataSetQualityFeature = !!capabilities?.data_quality; - const hasIndexManagementFeature = !!capabilities?.index_management; - const togglePopover = useCallback((streamName: string | null) => { setPopoverOpen((prev) => (prev === streamName ? null : streamName)); }, []); + return ( - {data.charts.map((chart, idx) => { - const chartTimestamps = chart.series.flatMap((series) => series.data.map((d) => d.x)); - const minTimestamp = Math.min(...chartTimestamps); - const maxTimestamp = Math.max(...chartTimestamps); - const tickFormat = niceTimeFormatter([minTimestamp, maxTimestamp]); - - return ( - - -
- -
{chartKeyToTitleMap[chart.key] || chart.key}
- - - ( - - )} - /> - {chart.series.map((stream, streamIdx) => ( - [point.x, point.y])} - xScaleType={ScaleType.Time} - yScaleType={ScaleType.Linear} - xAccessor={0} - yAccessors={[1]} - stackAccessors={[0]} - /> - ))} - - - - formatBytes(d)} - /> - -
- - - ); - })} + {data.charts.map((chart, idx) => ( + + ))} ); }; diff --git a/x-pack/plugins/data_usage/public/app/components/legend_action.tsx b/x-pack/plugins/data_usage/public/app/components/legend_action.tsx index 611c8acf0f320..a816d1f8eadda 100644 --- a/x-pack/plugins/data_usage/public/app/components/legend_action.tsx +++ b/x-pack/plugins/data_usage/public/app/components/legend_action.tsx @@ -21,28 +21,22 @@ interface LegendActionProps { idx: number; popoverOpen: string | null; togglePopover: (streamName: string | null) => void; - hasIndexManagementFeature: boolean; - hasDataSetQualityFeature: boolean; label: string; } export const LegendAction: React.FC = React.memo( - ({ - label, - idx, - popoverOpen, - togglePopover, - hasIndexManagementFeature, - hasDataSetQualityFeature, - }) => { + ({ label, idx, popoverOpen, togglePopover }) => { const uniqueStreamName = `${idx}-${label}`; const { services: { share: { url: { locators }, }, + application: { capabilities }, }, } = useKibanaContextForPlugin(); + const hasDataSetQualityFeature = !!capabilities?.data_quality; + const hasIndexManagementFeature = !!capabilities?.index_management; const onClickIndexManagement = useCallback(async () => { // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 From 34c15c85ee564a90d6bab4d96bb15c476e6714e7 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 09:30:55 -0400 Subject: [PATCH 42/53] pass datepicker state to dataset quality link --- .../public/app/components/dataset_quality_link.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx index d71093be9bb4a..00f9ea1a1e6a8 100644 --- a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -9,6 +9,7 @@ import React from 'react'; import { EuiListGroupItem } from '@elastic/eui'; import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +import { useDateRangePicker } from '../hooks/use_date_picker'; interface DatasetQualityLinkProps { dataStreamName: string; @@ -16,19 +17,18 @@ interface DatasetQualityLinkProps { export const DatasetQualityLink: React.FC = React.memo( ({ dataStreamName }) => { + const { dateRangePickerState } = useDateRangePicker(); const { services: { share: { url }, }, } = useKibanaContextForPlugin(); - + const { startDate, endDate } = dateRangePickerState; const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); - const onClickDataQuality = async () => { const locatorParams: DataQualityLocatorParams = { filters: { - // TODO: get time range from our page state - timeRange: { from: 'now-15m', to: 'now', refresh: { pause: true, value: 0 } }, + timeRange: { from: startDate, to: endDate, refresh: { pause: true, value: 0 } }, }, }; const dataset = getDatasetFromDataStream(dataStreamName); From 1265c9fdd98ce0209347df95347f5bac8228a5ad Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:22:47 +0000 Subject: [PATCH 43/53] [CI] Auto-commit changed files from 'node scripts/notice' --- x-pack/plugins/data_usage/tsconfig.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugins/data_usage/tsconfig.json b/x-pack/plugins/data_usage/tsconfig.json index 9087062db0b06..cecbeb654db30 100644 --- a/x-pack/plugins/data_usage/tsconfig.json +++ b/x-pack/plugins/data_usage/tsconfig.json @@ -22,6 +22,12 @@ "@kbn/share-plugin", "@kbn/config-schema", "@kbn/logging", + "@kbn/deeplinks-observability", + "@kbn/unified-search-plugin", + "@kbn/i18n-react", + "@kbn/core-http-browser", + "@kbn/core-chrome-browser", + "@kbn/features-plugin", ], "exclude": ["target/**/*"] } From dcdfca9bef4784addc72f314bd56e4d9528ef7ac Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Mon, 7 Oct 2024 17:54:25 +0200 Subject: [PATCH 44/53] remove redundant file --- x-pack/plugins/data_usage/server/utils/index.ts | 1 - .../server/utils/validate_time_range.ts | 17 ----------------- 2 files changed, 18 deletions(-) delete mode 100644 x-pack/plugins/data_usage/server/utils/validate_time_range.ts diff --git a/x-pack/plugins/data_usage/server/utils/index.ts b/x-pack/plugins/data_usage/server/utils/index.ts index 78b123f7c2bdf..af46a18f61a79 100644 --- a/x-pack/plugins/data_usage/server/utils/index.ts +++ b/x-pack/plugins/data_usage/server/utils/index.ts @@ -5,5 +5,4 @@ * 2.0. */ -export { isValidateTimeRange } from './validate_time_range'; export { CustomHttpRequestError } from './custom_http_request_error'; diff --git a/x-pack/plugins/data_usage/server/utils/validate_time_range.ts b/x-pack/plugins/data_usage/server/utils/validate_time_range.ts deleted file mode 100644 index 8a311c991e9fa..0000000000000 --- a/x-pack/plugins/data_usage/server/utils/validate_time_range.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -export const isValidateTimeRange = (fromTimestamp: number, endTimestamp: number) => { - if ( - fromTimestamp < endTimestamp && - fromTimestamp > 0 && - endTimestamp > 0 && - fromTimestamp !== endTimestamp - ) { - return true; - } - return false; -}; From c78671416eae66ece703b407f00b51fb6497d6c0 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 12:01:00 -0400 Subject: [PATCH 45/53] use data quality details locator --- .../observability/locators/dataset_quality.ts | 1 - .../app/components/dataset_quality_link.tsx | 28 +++++++------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/packages/deeplinks/observability/locators/dataset_quality.ts b/packages/deeplinks/observability/locators/dataset_quality.ts index 6ae42f3805381..9a6dd85ade2d2 100644 --- a/packages/deeplinks/observability/locators/dataset_quality.ts +++ b/packages/deeplinks/observability/locators/dataset_quality.ts @@ -27,7 +27,6 @@ type TimeRangeConfig = { // eslint-disable-next-line @typescript-eslint/consistent-type-definitions type Filters = { timeRange: TimeRangeConfig; - query?: string; }; export interface DataQualityLocatorParams extends SerializableRecord { diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx index 00f9ea1a1e6a8..d6627f3d8dca2 100644 --- a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -7,7 +7,10 @@ import React from 'react'; import { EuiListGroupItem } from '@elastic/eui'; -import { DataQualityLocatorParams, DATA_QUALITY_LOCATOR_ID } from '@kbn/deeplinks-observability'; +import { + DataQualityDetailsLocatorParams, + DATA_QUALITY_DETAILS_LOCATOR_ID, +} from '@kbn/deeplinks-observability'; import { useKibanaContextForPlugin } from '../../utils/use_kibana'; import { useDateRangePicker } from '../hooks/use_date_picker'; @@ -24,17 +27,14 @@ export const DatasetQualityLink: React.FC = React.memo( }, } = useKibanaContextForPlugin(); const { startDate, endDate } = dateRangePickerState; - const locator = url.locators.get(DATA_QUALITY_LOCATOR_ID); + const locator = url.locators.get( + DATA_QUALITY_DETAILS_LOCATOR_ID + ); const onClickDataQuality = async () => { - const locatorParams: DataQualityLocatorParams = { - filters: { - timeRange: { from: startDate, to: endDate, refresh: { pause: true, value: 0 } }, - }, + const locatorParams: DataQualityDetailsLocatorParams = { + dataStream: dataStreamName, + timeRange: { from: startDate, to: endDate, refresh: { pause: true, value: 0 } }, }; - const dataset = getDatasetFromDataStream(dataStreamName); - if (locatorParams?.filters && dataset) { - locatorParams.filters.query = dataset; - } if (locator) { await locator.navigate(locatorParams); } @@ -42,11 +42,3 @@ export const DatasetQualityLink: React.FC = React.memo( return ; } ); - -function getDatasetFromDataStream(dataStreamName: string): string | null { - const parts = dataStreamName.split('-'); - if (parts.length !== 3) { - return null; - } - return parts[1]; -} From 720e8a14aa0437e600a08f651c039f27f74d686b Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 15:24:52 -0400 Subject: [PATCH 46/53] update mock data to align with autoOps response --- .../common/rest_types/usage_metrics.ts | 39 ++- .../public/app/components/chart_panel.tsx | 26 +- .../public/app/components/charts.tsx | 8 +- x-pack/plugins/data_usage/public/app/types.ts | 22 +- .../routes/internal/usage_metrics_handler.ts | 235 +++++++++--------- 5 files changed, 167 insertions(+), 163 deletions(-) diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 0c7971686ea43..9a53ed9671f35 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -28,14 +28,12 @@ export const isMetricType = (type: string): type is MetricTypes => // @ts-ignore const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); -const DateSchema = schema.maybe(schema.string()); +const DateSchema = schema.string(); -const metricTypesSchema = { +const metricTypesSchema = schema.oneOf( // @ts-expect-error TS2769: No overload matches this call - schema: schema.oneOf(METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType))), - options: { minSize: 1, maxSize: METRIC_TYPE_VALUES.length }, -}; - + METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType)) // Create a oneOf schema for the keys +); export const UsageMetricsRequestSchema = { query: schema.object({ from: DateSchema, @@ -43,7 +41,7 @@ export const UsageMetricsRequestSchema = { size: schema.maybe(schema.number()), // should be same as dataStreams.length metricTypes: schema.oneOf([ schema.arrayOf(schema.string(), { - ...metricTypesSchema.options, + minSize: 1, validate: (values) => { if (values.map((v) => v.trim()).some((v) => !v.length)) { return '[metricTypes] list can not contain empty values'; @@ -86,24 +84,17 @@ export type UsageMetricsRequestSchemaQueryParams = TypeOf schema.object({ - charts: schema.arrayOf( - schema.object({ - key: metricTypesSchema.schema, - series: schema.arrayOf( - schema.object({ - streamName: schema.string(), - data: schema.arrayOf( - schema.object({ - x: schema.number(), - y: schema.number(), - }) - ), - }) - ), - }), - { maxSize: 2 } + metrics: schema.recordOf( + metricTypesSchema, + schema.arrayOf( + schema.object({ + name: schema.string(), + data: schema.arrayOf( + schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 }) // Each data point is an array of 2 numbers + ), + }) + ) ), }), }; - export type UsageMetricsResponseSchemaBody = TypeOf; diff --git a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx index e17ad2692c63a..c7937ae149de9 100644 --- a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx +++ b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx @@ -20,7 +20,7 @@ import { import { i18n } from '@kbn/i18n'; import { LegendAction } from './legend_action'; import { MetricTypes } from '../../../common/rest_types'; -import { Chart as ChartData } from '../types'; +import { MetricSeries } from '../types'; // TODO: Remove this when we have a title for each metric type type ChartKey = Extract; @@ -34,21 +34,23 @@ export const chartKeyToTitleMap: Record = { }; interface ChartPanelProps { - data: ChartData; + metricType: MetricTypes; + series: MetricSeries[]; idx: number; popoverOpen: string | null; togglePopover: (streamName: string | null) => void; } export const ChartPanel: React.FC = ({ - data, + metricType, + series, idx, popoverOpen, togglePopover, }) => { const theme = useEuiTheme(); - const chartTimestamps = data.series.flatMap((series) => series.data.map((d) => d.x)); + const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d[0])); const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)]; @@ -71,10 +73,10 @@ export const ChartPanel: React.FC = ({ [idx, popoverOpen, togglePopover] ); return ( - + -
{chartKeyToTitleMap[data.key as ChartKey] || data.key}
+
{chartKeyToTitleMap[metricType as ChartKey] || metricType}
 = ({ xDomain={{ min: minTimestamp, max: maxTimestamp }} legendAction={renderLegendAction} /> - {data.series.map((stream, streamIdx) => ( + {series.map((stream, streamIdx) => ( [point.x, point.y])} + id={`${metricType}-${stream.name}`} + name={stream.name} + data={stream.data} xScaleType={ScaleType.Time} yScaleType={ScaleType.Linear} - xAccessor={0} - yAccessors={[1]} + xAccessor={0} // x is the first element in the tuple + yAccessors={[1]} // y is the second element in the tuple stackAccessors={[0]} /> ))} diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx index ccf753d579f20..6549f7e03830a 100644 --- a/x-pack/plugins/data_usage/public/app/components/charts.tsx +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -7,6 +7,7 @@ import React, { useCallback, useState } from 'react'; import { EuiFlexGroup } from '@elastic/eui'; import { MetricsResponse } from '../types'; +import { MetricTypes } from '../../../common/rest_types'; import { ChartPanel } from './chart_panel'; interface ChartsProps { data: MetricsResponse; @@ -20,10 +21,11 @@ export const Charts: React.FC = ({ data }) => { return ( - {data.charts.map((chart, idx) => ( + {Object.entries(data.metrics).map(([metricType, series], idx) => ( >; export interface MetricsResponse { - charts: Chart[]; + metrics: Metrics; +} +export interface MetricsResponse { + metrics: Metrics; } diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 6ae9b0ffbe82d..91cd1ff044e59 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -7,7 +7,11 @@ import { RequestHandler } from '@kbn/core/server'; import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; -import { MetricTypes, UsageMetricsRequestSchemaQueryParams } from '../../../common/rest_types'; +import { + MetricTypes, + UsageMetricsRequestSchemaQueryParams, + UsageMetricsResponseSchema, +} from '../../../common/rest_types'; import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; import { errorHandler } from '../error_handler'; @@ -53,12 +57,12 @@ export const getUsageMetricsHandler = ( if (!userDsNames.length) { return response.ok({ body: { - charts: [], + metrics: {}, }, }); } - const charts = await fetchMetricsFromAutoOps({ + const metrics = await fetchMetricsFromAutoOps({ from, to, size, @@ -68,10 +72,11 @@ export const getUsageMetricsHandler = ( return response.ok({ body: { - charts, + metrics, }, }); } catch (error) { + logger.error(`Error retrieving usage metrics: ${error.message}`); return errorHandler(logger, response, error); } }; @@ -91,140 +96,146 @@ const fetchMetricsFromAutoOps = async ({ dataStreams: string[]; }) => { // TODO: fetch data from autoOps using userDsNames - - const charts = [ - { - key: 'ingest_rate', - series: [ + /* + const response = await axios.post('https://api.auto-ops.{region}.{csp}.cloud.elastic.co/monitoring/serverless/v1/projects/{project_id}/metrics', { + from: Date.parse(from), + to: Date.parse(to), + metric_types: metricTypes, + allowed_indices: dataStreams, + size: size || 10, + }); + const { data } = response;*/ + // mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml + const mockData = { + metrics: { + ingest_rate: [ { - streamName: 'data_stream_1', + name: 'metrics-apache_spark.driver-default', data: [ - { x: 1726858530000, y: 13756849 }, - { x: 1726862130000, y: 14657904 }, - { x: 1726865730000, y: 12798561 }, - { x: 1726869330000, y: 13578213 }, - { x: 1726872930000, y: 14123495 }, - { x: 1726876530000, y: 13876548 }, - { x: 1726880130000, y: 12894561 }, - { x: 1726883730000, y: 14478953 }, - { x: 1726887330000, y: 14678905 }, - { x: 1726890930000, y: 13976547 }, - { x: 1726894530000, y: 14568945 }, - { x: 1726898130000, y: 13789561 }, - { x: 1726901730000, y: 14478905 }, - { x: 1726905330000, y: 13956423 }, - { x: 1726908930000, y: 14598234 }, + [1726858530000, 13756849], + [1726862130000, 14657904], + [1726865730000, 12798561], + [1726869330000, 13578213], + [1726872930000, 14123495], + [1726876530000, 13876548], + [1726880130000, 12894561], + [1726883730000, 14478953], + [1726887330000, 14678905], + [1726890930000, 13976547], + [1726894530000, 14568945], + [1726898130000, 13789561], + [1726901730000, 14478905], + [1726905330000, 13956423], + [1726908930000, 14598234], ], }, { - streamName: 'data_stream_2', + name: 'logs-apm.app.adservice-default', data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - { x: 1726869330000, y: 14048532 }, - { x: 1726872930000, y: 14237495 }, - { x: 1726876530000, y: 13745689 }, - { x: 1726880130000, y: 13974562 }, - { x: 1726883730000, y: 14234653 }, - { x: 1726887330000, y: 14323479 }, - { x: 1726890930000, y: 14023945 }, - { x: 1726894530000, y: 14189673 }, - { x: 1726898130000, y: 14247895 }, - { x: 1726901730000, y: 14098324 }, - { x: 1726905330000, y: 14478905 }, - { x: 1726908930000, y: 14323894 }, + [1726858530000, 12894623], + [1726862130000, 14436905], + [1726865730000, 13794805], + [1726869330000, 14048532], + [1726872930000, 14237495], + [1726876530000, 13745689], + [1726880130000, 13974562], + [1726883730000, 14234653], + [1726887330000, 14323479], + [1726890930000, 14023945], + [1726894530000, 14189673], + [1726898130000, 14247895], + [1726901730000, 14098324], + [1726905330000, 14478905], + [1726908930000, 14323894], ], }, { - streamName: 'data_stream_3', + name: 'metrics-apm.app.aws-lambdas-default', data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], ], }, ], - }, - { - key: 'storage_retained', - series: [ + storage_retained: [ { - streamName: 'data_stream_1', + name: 'metrics-apache_spark.driver-default', data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], ], }, { - streamName: 'data_stream_2', + name: 'logs-apm.app.adservice-default', data: [ - { x: 1726858530000, y: 12894623 }, - { x: 1726862130000, y: 14436905 }, - { x: 1726865730000, y: 13794805 }, - { x: 1726869330000, y: 14048532 }, - { x: 1726872930000, y: 14237495 }, - { x: 1726876530000, y: 13745689 }, - { x: 1726880130000, y: 13974562 }, - { x: 1726883730000, y: 14234653 }, - { x: 1726887330000, y: 14323479 }, - { x: 1726890930000, y: 14023945 }, - { x: 1726894530000, y: 14189673 }, - { x: 1726898130000, y: 14247895 }, - { x: 1726901730000, y: 14098324 }, - { x: 1726905330000, y: 14478905 }, - { x: 1726908930000, y: 14323894 }, + [1726858530000, 12894623], + [1726862130000, 14436905], + [1726865730000, 13794805], + [1726869330000, 14048532], + [1726872930000, 14237495], + [1726876530000, 13745689], + [1726880130000, 13974562], + [1726883730000, 14234653], + [1726887330000, 14323479], + [1726890930000, 14023945], + [1726894530000, 14189673], + [1726898130000, 14247895], + [1726901730000, 14098324], + [1726905330000, 14478905], + [1726908930000, 14323894], ], }, { - streamName: 'data_stream_3', + name: 'metrics-apm.app.aws-lambdas-default', data: [ - { x: 1726858530000, y: 12576413 }, - { x: 1726862130000, y: 13956423 }, - { x: 1726865730000, y: 14568945 }, - { x: 1726869330000, y: 14234856 }, - { x: 1726872930000, y: 14368942 }, - { x: 1726876530000, y: 13897654 }, - { x: 1726880130000, y: 14456989 }, - { x: 1726883730000, y: 14568956 }, - { x: 1726887330000, y: 13987562 }, - { x: 1726890930000, y: 14567894 }, - { x: 1726894530000, y: 14246789 }, - { x: 1726898130000, y: 14567895 }, - { x: 1726901730000, y: 14457896 }, - { x: 1726905330000, y: 14567895 }, - { x: 1726908930000, y: 13989456 }, + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], ], }, - // Repeat similar structure for more data streams... ], }, - ]; + }; + // Make sure data is what we expect + const validatedData = UsageMetricsResponseSchema.body().validate(mockData); - return charts; + return validatedData.metrics; }; From d04364374110a376f281351642c314bb0b30d5f3 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 15:36:19 -0400 Subject: [PATCH 47/53] remove size for now --- x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts | 1 - .../plugins/data_usage/public/hooks/use_get_usage_metrics.ts | 1 - .../server/routes/internal/usage_metrics_handler.ts | 4 ---- 3 files changed, 6 deletions(-) diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 9a53ed9671f35..54f84189541c8 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -38,7 +38,6 @@ export const UsageMetricsRequestSchema = { query: schema.object({ from: DateSchema, to: DateSchema, - size: schema.maybe(schema.number()), // should be same as dataStreams.length metricTypes: schema.oneOf([ schema.arrayOf(schema.string(), { minSize: 1, diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts index 91cee020f243e..6b9860e997c12 100644 --- a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -37,7 +37,6 @@ export const useGetDataUsageMetrics = ( from: query.from, to: query.to, metricTypes: query.metricTypes, - size: query.size, dataStreams: query.dataStreams, }, }); diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts index 91cd1ff044e59..6f992c9fb2a38 100644 --- a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -65,7 +65,6 @@ export const getUsageMetricsHandler = ( const metrics = await fetchMetricsFromAutoOps({ from, to, - size, metricTypes: formatStringParams(metricTypes) as MetricTypes[], dataStreams: formatStringParams(userDsNames), }); @@ -85,13 +84,11 @@ export const getUsageMetricsHandler = ( const fetchMetricsFromAutoOps = async ({ from, to, - size, metricTypes, dataStreams, }: { from: string; to: string; - size?: number; metricTypes: MetricTypes[]; dataStreams: string[]; }) => { @@ -102,7 +99,6 @@ const fetchMetricsFromAutoOps = async ({ to: Date.parse(to), metric_types: metricTypes, allowed_indices: dataStreams, - size: size || 10, }); const { data } = response;*/ // mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml From 889018fa31133195dbc632585e8a160155c08458 Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 18:56:23 -0400 Subject: [PATCH 48/53] fix types --- x-pack/plugins/data_usage/public/app/data_usage.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index 77104eeb1d5c3..f9b26cb132969 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -26,6 +26,7 @@ import { PLUGIN_NAME } from '../../common'; import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; import { useDateRangePicker } from './hooks/use_date_picker'; import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params'; +import { MetricsResponse } from './types'; export const DataUsage = () => { const { @@ -38,6 +39,8 @@ export const DataUsage = () => { const [queryParams, setQueryParams] = useState({ metricTypes: ['storage_retained', 'ingest_rate'], dataStreams: [], + from: 'now-24h/h', + to: 'now', }); useEffect(() => { @@ -109,7 +112,7 @@ export const DataUsage = () => { - {isFetched && data ? : } + {isFetched && data ? : } ); From 222c6db6b786b77bcb4bbd6d7c79c8432417857f Mon Sep 17 00:00:00 2001 From: neptunian Date: Mon, 7 Oct 2024 21:48:48 -0400 Subject: [PATCH 49/53] fix test --- .../common/rest_types/usage_metrics.test.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts index 1e04c84c113d1..c887ba646110a 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts @@ -11,6 +11,8 @@ describe('usage_metrics schemas', () => { it('should accept valid request query', () => { expect(() => UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), metricTypes: ['storage_retained'], }) ).not.toThrow(); @@ -47,24 +49,12 @@ describe('usage_metrics schemas', () => { ).not.toThrow(); }); - it('should accept valid `size`', () => { - expect(() => - UsageMetricsRequestSchema.query.validate({ - from: new Date().toISOString(), - to: new Date().toISOString(), - metricTypes: ['storage_retained'], - size: 100, - }) - ).not.toThrow(); - }); - it('should accept `dataStream` list', () => { expect(() => UsageMetricsRequestSchema.query.validate({ from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], - size: 3, dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], }) ).not.toThrow(); @@ -76,7 +66,6 @@ describe('usage_metrics schemas', () => { from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], - size: 3, dataStreams: [], }) ).toThrowError('expected value of type [string] but got [Array]'); @@ -88,7 +77,6 @@ describe('usage_metrics schemas', () => { from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], - size: 1, dataStreams: ' ', }) ).toThrow('[dataStreams] must have at least one value'); @@ -100,7 +88,6 @@ describe('usage_metrics schemas', () => { from: new Date().toISOString(), to: new Date().toISOString(), metricTypes: ['storage_retained'], - size: 3, dataStreams: ['ds_1', ' '], }) ).toThrow('[dataStreams] list can not contain empty values'); From 9c2fc1a1593e33218cde881204525d75e87ce2e0 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 8 Oct 2024 10:26:33 +0200 Subject: [PATCH 50/53] remove redundant const --- x-pack/plugins/data_usage/public/app/data_usage.tsx | 6 +++--- .../public/app/hooks/use_charts_url_params.tsx | 11 +---------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index f9b26cb132969..b8d036392622b 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -24,7 +24,7 @@ import { useBreadcrumbs } from '../utils/use_breadcrumbs'; import { useKibanaContextForPlugin } from '../utils/use_kibana'; import { PLUGIN_NAME } from '../../common'; import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; -import { useDateRangePicker } from './hooks/use_date_picker'; +import { DEFAULT_DATE_RANGE_OPTIONS, useDateRangePicker } from './hooks/use_date_picker'; import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params'; import { MetricsResponse } from './types'; @@ -39,8 +39,8 @@ export const DataUsage = () => { const [queryParams, setQueryParams] = useState({ metricTypes: ['storage_retained', 'ingest_rate'], dataStreams: [], - from: 'now-24h/h', - to: 'now', + from: DEFAULT_DATE_RANGE_OPTIONS.startDate, + to: DEFAULT_DATE_RANGE_OPTIONS.endDate, }); useEffect(() => { diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx index 9b31e10b9e18e..4626c81d1192e 100644 --- a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx +++ b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx @@ -8,6 +8,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { useHistory, useLocation } from 'react-router-dom'; import { MetricTypes, isMetricType } from '../../../common/rest_types'; import { useUrlParams } from '../../hooks/use_url_params'; +import { DEFAULT_DATE_RANGE_OPTIONS } from './use_date_picker'; interface UrlParamsDataUsageMetricsFilters { metricTypes: string; @@ -31,16 +32,6 @@ type FiltersFromUrl = Pick< 'metricTypes' | 'dataStreams' | 'startDate' | 'endDate' >; -export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({ - autoRefreshOptions: { - enabled: false, - duration: 10000, - }, - startDate: 'now-24h/h', - endDate: 'now', - recentlyUsedDateRanges: [], -}); - export const getDataUsageMetricsFiltersFromUrlParams = ( urlParams: Partial ): FiltersFromUrl => { From 8f639d7d8adecb6a3e665d50754f80bec37f3e21 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 8 Oct 2024 11:31:22 +0200 Subject: [PATCH 51/53] update tests for API schema --- .../common/rest_types/usage_metrics.test.ts | 40 +++++++++++++++++++ .../common/rest_types/usage_metrics.ts | 5 ++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts index c887ba646110a..f6c08e2caddc0 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts @@ -136,4 +136,44 @@ describe('usage_metrics schemas', () => { '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' ); }); + + it('should error if `from` is not a valid input', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: 1010, + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[from]: expected value of type [string] but got [number]'); + }); + + it('should error if `to` is not a valid input', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: 1010, + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[to]: expected value of type [string] but got [number]'); + }); + + it('should error if `from` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: ' ', + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[from]: Date ISO string must not be empty'); + }); + + it('should error if `to` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: ' ', + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[to]: Date ISO string must not be empty'); + }); }); diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts index 54f84189541c8..f2bbdb616fc79 100644 --- a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -28,7 +28,10 @@ export const isMetricType = (type: string): type is MetricTypes => // @ts-ignore const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); -const DateSchema = schema.string(); +const DateSchema = schema.string({ + minLength: 1, + validate: (v) => (v.trim().length ? undefined : 'Date ISO string must not be empty'), +}); const metricTypesSchema = schema.oneOf( // @ts-expect-error TS2769: No overload matches this call From 297091d7b6f5eba237e2af27d36bfc1ce83b14ca Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 8 Oct 2024 11:31:49 +0200 Subject: [PATCH 52/53] cleanup --- .../public/app/hooks/use_charts_url_params.tsx | 11 +++++------ .../plugins/data_usage/server/routes/error_handler.ts | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx index 4626c81d1192e..0e03da5d9adbd 100644 --- a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx +++ b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx @@ -43,15 +43,14 @@ export const getDataUsageMetricsFiltersFromUrlParams = ( }; const urlMetricTypes = urlParams.metricTypes - ? (String(urlParams.metricTypes).split(',') as MetricTypes[]).reduce( - (acc, curr) => { + ? String(urlParams.metricTypes) + .split(',') + .reduce((acc, curr) => { if (isMetricType(curr)) { acc.push(curr); } return acc.sort(); - }, - [] - ) + }, []) : []; const urlDataStreams = urlParams.dataStreams @@ -85,7 +84,7 @@ export const useDataUsageMetricsUrlParams = (): DataUsageMetricsFiltersFromUrlPa ...location, search: toUrlParams({ ...urlParams, - hosts: metricTypes.length ? metricTypes : undefined, + metricTypes: metricTypes.length ? metricTypes : undefined, }), }); }, diff --git a/x-pack/plugins/data_usage/server/routes/error_handler.ts b/x-pack/plugins/data_usage/server/routes/error_handler.ts index 7bd774f2e71e5..122df5e72b130 100644 --- a/x-pack/plugins/data_usage/server/routes/error_handler.ts +++ b/x-pack/plugins/data_usage/server/routes/error_handler.ts @@ -12,7 +12,7 @@ import { BaseError } from '../common/errors'; export class NotFoundError extends BaseError {} /** - * Default Endpoint Routes error handler + * Default Data Usage Routes error handler * @param logger * @param res * @param error From 3b39d20d87c75052e396687434c61cf44d35f403 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Tue, 8 Oct 2024 13:27:20 +0200 Subject: [PATCH 53/53] set default URL params for metrics and date ranges --- .../data_usage/public/app/data_usage.tsx | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx index b8d036392622b..c32f86d68b5bf 100644 --- a/x-pack/plugins/data_usage/public/app/data_usage.tsx +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -33,8 +33,14 @@ export const DataUsage = () => { services: { chrome, appParams }, } = useKibanaContextForPlugin(); - const { metricTypes: metricTypesFromUrl, dataStreams: dataStreamsFromUrl } = - useDataUsageMetricsUrlParams(); + const { + metricTypes: metricTypesFromUrl, + dataStreams: dataStreamsFromUrl, + startDate: startDateFromUrl, + endDate: endDateFromUrl, + setUrlMetricTypesFilter, + setUrlDateRangeFilter, + } = useDataUsageMetricsUrlParams(); const [queryParams, setQueryParams] = useState({ metricTypes: ['storage_retained', 'ingest_rate'], @@ -43,6 +49,28 @@ export const DataUsage = () => { to: DEFAULT_DATE_RANGE_OPTIONS.endDate, }); + useEffect(() => { + if (!metricTypesFromUrl) { + setUrlMetricTypesFilter( + typeof queryParams.metricTypes !== 'string' + ? queryParams.metricTypes.join(',') + : queryParams.metricTypes + ); + } + if (!startDateFromUrl || !endDateFromUrl) { + setUrlDateRangeFilter({ startDate: queryParams.from, endDate: queryParams.to }); + } + }, [ + endDateFromUrl, + metricTypesFromUrl, + queryParams.from, + queryParams.metricTypes, + queryParams.to, + setUrlDateRangeFilter, + setUrlMetricTypesFilter, + startDateFromUrl, + ]); + useEffect(() => { setQueryParams((prevState) => ({ ...prevState,