-
Notifications
You must be signed in to change notification settings - Fork 8.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Fleet] Use metering API in serverless #200063
Changes from all commits
d1cb8c1
db52a95
fc3f72e
6e46134
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,7 @@ | |
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
import type { Dictionary } from 'lodash'; | ||
import { keyBy, keys, merge } from 'lodash'; | ||
import type { RequestHandler } from '@kbn/core/server'; | ||
import pMap from 'p-map'; | ||
|
@@ -13,9 +14,13 @@ import { KibanaSavedObjectType } from '../../../common/types'; | |
import type { GetDataStreamsResponse } from '../../../common/types'; | ||
import { getPackageSavedObjects } from '../../services/epm/packages/get'; | ||
import { defaultFleetErrorHandler } from '../../errors'; | ||
import type { MeteringStats } from '../../services/data_streams'; | ||
import { dataStreamService } from '../../services/data_streams'; | ||
|
||
import { getDataStreamsQueryMetadata } from './get_data_streams_query_metadata'; | ||
import type { IndicesDataStreamsStatsDataStreamsStatsItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; | ||
import { ByteSizeValue } from '@kbn/config-schema'; | ||
import { appContextService } from '../../services'; | ||
|
||
const MANAGED_BY = 'fleet'; | ||
const LEGACY_MANAGED_BY = 'ingest-manager'; | ||
|
@@ -51,10 +56,22 @@ export const getListHandler: RequestHandler = async (context, request, response) | |
}; | ||
|
||
try { | ||
const useMeteringApi = appContextService.getConfig()?.internal?.useMeteringApi; | ||
|
||
// Get matching data streams, their stats, and package SOs | ||
const [dataStreamsInfo, dataStreamStats, packageSavedObjects] = await Promise.all([ | ||
const [ | ||
dataStreamsInfo, | ||
dataStreamStatsOrUndefined, | ||
dataStreamMeteringStatsorUndefined, | ||
packageSavedObjects, | ||
] = await Promise.all([ | ||
dataStreamService.getAllFleetDataStreams(esClient), | ||
dataStreamService.getAllFleetDataStreamsStats(esClient), | ||
useMeteringApi | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: we could simplify by moving these out of |
||
? undefined | ||
: dataStreamService.getAllFleetDataStreamsStats(elasticsearch.client.asSecondaryAuthUser), | ||
useMeteringApi | ||
? dataStreamService.getAllFleetMeteringStats(elasticsearch.client.asSecondaryAuthUser) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The API is only available as internal, and calling it as current user fail for that reason. |
||
: undefined, | ||
getPackageSavedObjects(savedObjects.client), | ||
]); | ||
|
||
|
@@ -67,13 +84,24 @@ export const getListHandler: RequestHandler = async (context, request, response) | |
|
||
const dataStreamsInfoByName = keyBy<ESDataStreamInfo>(filteredDataStreamsInfo, 'name'); | ||
|
||
const filteredDataStreamsStats = dataStreamStats.filter( | ||
(dss) => !!dataStreamsInfoByName[dss.data_stream] | ||
); | ||
const dataStreamsStatsByName = keyBy(filteredDataStreamsStats, 'data_stream'); | ||
let dataStreamsStatsByName: Dictionary<IndicesDataStreamsStatsDataStreamsStatsItem> = {}; | ||
if (dataStreamStatsOrUndefined) { | ||
const filteredDataStreamsStats = dataStreamStatsOrUndefined.filter( | ||
(dss) => !!dataStreamsInfoByName[dss.data_stream] | ||
); | ||
dataStreamsStatsByName = keyBy(filteredDataStreamsStats, 'data_stream'); | ||
} | ||
let dataStreamsMeteringStatsByName: Dictionary<MeteringStats> = {}; | ||
if (dataStreamMeteringStatsorUndefined) { | ||
dataStreamsMeteringStatsByName = keyBy(dataStreamMeteringStatsorUndefined, 'name'); | ||
} | ||
|
||
// Combine data stream info | ||
const dataStreams = merge(dataStreamsInfoByName, dataStreamsStatsByName); | ||
const dataStreams = merge( | ||
dataStreamsInfoByName, | ||
dataStreamsStatsByName, | ||
dataStreamsMeteringStatsByName | ||
); | ||
const dataStreamNames = keys(dataStreams); | ||
|
||
// Map package SOs | ||
|
@@ -132,10 +160,14 @@ export const getListHandler: RequestHandler = async (context, request, response) | |
package: dataStream._meta?.package?.name || '', | ||
package_version: '', | ||
last_activity_ms: dataStream.maximum_timestamp, // overridden below if maxIngestedTimestamp agg returns a result | ||
size_in_bytes: dataStream.store_size_bytes, | ||
size_in_bytes: dataStream.store_size_bytes || dataStream.size_in_bytes, | ||
// `store_size` should be available from ES due to ?human=true flag | ||
// but fallback to bytes just in case | ||
size_in_bytes_formatted: dataStream.store_size || `${dataStream.store_size_bytes}b`, | ||
size_in_bytes_formatted: | ||
dataStream.store_size || | ||
new ByteSizeValue( | ||
dataStream.store_size_bytes || dataStream.size_in_bytes || 0 | ||
).toString(), | ||
dashboards: [], | ||
serviceDetails: null, | ||
}; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm wondering if we could use the metering API in stateful too, or is it only for serverless?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Unfortunately it's only available in serverless