diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts index 72568fbd4c8a0..08b0ed72da89d 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts @@ -21,6 +21,15 @@ export const cloudSecurityMetringCallback = async ({ lastSuccessfulReport, config, }: MeteringCallbackInput): Promise => { + const projectHasCloudProductLine = config.productTypes.some( + (product) => product.product_line === ProductLine.cloud + ); + + if (!projectHasCloudProductLine) { + logger.info('No cloud product line found in the project'); + return { records: [] }; + } + const projectId = cloudSetup?.serverless?.projectId || 'missing_project_id'; const tier: Tier = getCloudProductTier(config, logger); diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts index 7827c5d25ebe6..3f141810747cd 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts @@ -15,7 +15,6 @@ import { CSPM, KSPM, METERING_CONFIGS, - THRESHOLD_MINUTES, BILLABLE_ASSETS_CONFIG, } from './constants'; import type { Tier, UsageRecord } from '../types'; @@ -23,69 +22,105 @@ import type { CloudSecurityMeteringCallbackInput, CloudSecuritySolutions, AssetCountAggregation, + ResourceSubtypeAggregationBucket, } from './types'; +export interface ResourceSubtypeCounter { + [key: string]: { + doc_count: number; + unique_assets: number; + }; +} + export const getUsageRecords = ( - assetCountAggregations: AssetCountAggregation[], + assetCountAggregation: AssetCountAggregation, cloudSecuritySolution: CloudSecuritySolutions, taskId: string, tier: Tier, projectId: string, periodSeconds: number, logger: Logger -): UsageRecord[] => { - const usageRecords = assetCountAggregations.map((assetCountAggregation) => { - const assetCount = assetCountAggregation.unique_assets.value; - - if (assetCount > AGGREGATION_PRECISION_THRESHOLD) { - logger.warn( - `The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.` - ); - } - - const minTimestamp = new Date( - assetCountAggregation.min_timestamp.value_as_string - ).toISOString(); - - const creationTimestamp = new Date(); - const minutes = creationTimestamp.getMinutes(); - if (minutes >= 30) { - creationTimestamp.setMinutes(30, 0, 0); - } else { - creationTimestamp.setMinutes(0, 0, 0); - } - const roundedCreationTimestamp = creationTimestamp.toISOString(); - - const usageRecord: UsageRecord = { - id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`, - usage_timestamp: minTimestamp, - creation_timestamp: creationTimestamp.toISOString(), - usage: { - type: CLOUD_SECURITY_TASK_TYPE, - sub_type: cloudSecuritySolution, - quantity: assetCount, - period_seconds: periodSeconds, - }, - source: { - id: taskId, - instance_group_id: projectId, - metadata: { tier }, - }, - }; +): UsageRecord => { + const resourceSubtypeCounter = assetCountAggregation.resource_sub_type.buckets.reduce( + (resourceMap, item) => { + resourceMap[item.key] = { + doc_count: item.doc_count, + unique_assets: item.unique_assets.value, + }; + return resourceMap; + }, + {} as ResourceSubtypeCounter + ); + + const resourceSubtypeBuckets: ResourceSubtypeAggregationBucket[] = + assetCountAggregation.resource_sub_type.buckets; + + let assetCount; + + if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) { + const billableAssets = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values; + assetCount = resourceSubtypeBuckets + .filter((bucket) => billableAssets.includes(bucket.key)) + .reduce((acc, bucket) => acc + bucket.unique_assets.value, 0); + } else { + assetCount = resourceSubtypeBuckets.reduce( + (acc, bucket) => acc + bucket.unique_assets.value, + 0 + ); + } - return usageRecord; - }); - return usageRecords; + if (assetCount > AGGREGATION_PRECISION_THRESHOLD) { + logger.warn( + `The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.` + ); + } + + const minTimestamp = new Date(assetCountAggregation.min_timestamp.value_as_string).toISOString(); + + const creationTimestamp = new Date(); + const minutes = creationTimestamp.getMinutes(); + if (minutes >= 30) { + creationTimestamp.setMinutes(30, 0, 0); + } else { + creationTimestamp.setMinutes(0, 0, 0); + } + const roundedCreationTimestamp = creationTimestamp.toISOString(); + + const usageRecord: UsageRecord = { + id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`, + usage_timestamp: minTimestamp, + creation_timestamp: creationTimestamp.toISOString(), + usage: { + type: CLOUD_SECURITY_TASK_TYPE, + sub_type: cloudSecuritySolution, + quantity: assetCount, + period_seconds: periodSeconds, + }, + source: { + id: taskId, + instance_group_id: projectId, + metadata: { tier, resourceSubtypeCounter }, + }, + }; + + return usageRecord; }; export const getAggregationByCloudSecuritySolution = ( cloudSecuritySolution: CloudSecuritySolutions ) => { return { - unique_assets: { - cardinality: { - field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier, - precision_threshold: AGGREGATION_PRECISION_THRESHOLD, + resource_sub_type: { + terms: { + field: `resource.sub_type`, + }, + aggs: { + unique_assets: { + cardinality: { + field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier, + precision_threshold: AGGREGATION_PRECISION_THRESHOLD, + }, + }, }, }, min_timestamp: { @@ -97,8 +132,7 @@ export const getAggregationByCloudSecuritySolution = ( }; export const getSearchQueryByCloudSecuritySolution = ( - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { const mustFilters = []; @@ -117,20 +151,11 @@ export const getSearchQueryByCloudSecuritySolution = ( } if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) { - const billableAssetsConfig = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution]; - mustFilters.push({ term: { 'rule.benchmark.posture_type': cloudSecuritySolution, }, }); - - // filter in only billable assets - mustFilters.push({ - terms: { - [billableAssetsConfig.filter_attribute]: billableAssetsConfig.values, - }, - }); } return { @@ -141,10 +166,9 @@ export const getSearchQueryByCloudSecuritySolution = ( }; export const getAssetAggQueryByCloudSecuritySolution = ( - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { - const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom); + const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution); const aggs = getAggregationByCloudSecuritySolution(cloudSecuritySolution); return { @@ -157,28 +181,27 @@ export const getAssetAggQueryByCloudSecuritySolution = ( export const getAssetAggByCloudSecuritySolution = async ( esClient: ElasticsearchClient, - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date -): Promise => { - const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom); + cloudSecuritySolution: CloudSecuritySolutions +): Promise => { + const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution); const response = await esClient.search(assetsAggQuery); - if (!response.aggregations) return []; - return [response.aggregations]; + if (!response.aggregations) return; + + return response.aggregations; }; const indexHasDataInDateRange = async ( esClient: ElasticsearchClient, - cloudSecuritySolution: CloudSecuritySolutions, - searchFrom: Date + cloudSecuritySolution: CloudSecuritySolutions ) => { const response = await esClient.search( { index: METERING_CONFIGS[cloudSecuritySolution].index, size: 1, _source: false, - query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom), + query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution), }, { ignore: [404] } ); @@ -186,22 +209,6 @@ const indexHasDataInDateRange = async ( return response.hits?.hits.length > 0; }; -const getSearchStartDate = (lastSuccessfulReport: Date): Date => { - const initialDate = new Date(); - const thresholdDate = new Date(initialDate.getTime() - THRESHOLD_MINUTES * 60 * 1000); - - if (lastSuccessfulReport) { - const lastSuccessfulReportDate = new Date(lastSuccessfulReport); - - const searchFrom = - lastSuccessfulReport && lastSuccessfulReportDate > thresholdDate - ? lastSuccessfulReportDate - : thresholdDate; - return searchFrom; - } - return thresholdDate; -}; - export const getCloudSecurityUsageRecord = async ({ esClient, projectId, @@ -212,21 +219,20 @@ export const getCloudSecurityUsageRecord = async ({ logger, }: CloudSecurityMeteringCallbackInput): Promise => { try { - const searchFrom = getSearchStartDate(lastSuccessfulReport); - - if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution, searchFrom))) return; + if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution))) return; // const periodSeconds = Math.floor((new Date().getTime() - searchFrom.getTime()) / 1000); const periodSeconds = 1800; // Workaround to prevent overbilling by charging for a constant time window. The issue should be resolved in https://github.com/elastic/security-team/issues/9424. - const assetCountAggregations = await getAssetAggByCloudSecuritySolution( + const assetCountAggregation = await getAssetAggByCloudSecuritySolution( esClient, - cloudSecuritySolution, - searchFrom + cloudSecuritySolution ); + if (!assetCountAggregation) return []; + const usageRecords = await getUsageRecords( - assetCountAggregations, + assetCountAggregation, cloudSecuritySolution, taskId, tier, @@ -235,7 +241,7 @@ export const getCloudSecurityUsageRecord = async ({ logger ); - return usageRecords; + return [usageRecords]; } catch (err) { logger.error(`Failed to fetch ${cloudSecuritySolution} metering data ${err}`); } diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts index 62ded11d5ad1e..f12877c6e4dcd 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts @@ -14,11 +14,20 @@ export interface CloudDefendAssetCountAggregation { export interface AssetCountAggregationBucket { buckets: AssetCountAggregation[]; } + +export interface ResourceSubtypeAggregationBucket { + key: string; + doc_count: number; + unique_assets: { + value: number; + }; +} + export interface AssetCountAggregation { key_as_string: string; min_timestamp: MinTimestamp; - unique_assets: { - value: number; + resource_sub_type: { + buckets: ResourceSubtypeAggregationBucket[]; }; } diff --git a/x-pack/plugins/security_solution_serverless/server/types.ts b/x-pack/plugins/security_solution_serverless/server/types.ts index 7e84b418cf4cf..0565feb6a2ae5 100644 --- a/x-pack/plugins/security_solution_serverless/server/types.ts +++ b/x-pack/plugins/security_solution_serverless/server/types.ts @@ -24,6 +24,7 @@ import type { ServerlessPluginSetup } from '@kbn/serverless/server'; import type { ProductTier } from '../common/product'; import type { ServerlessSecurityConfig } from './config'; +import { ResourceSubtypeCounter } from './cloud_security/cloud_security_metering_task'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface SecuritySolutionServerlessPluginSetup {} @@ -74,6 +75,7 @@ export interface UsageSource { export interface UsageSourceMetadata { tier?: Tier; + resourceSubtypeCounter?: ResourceSubtypeCounter; } export type Tier = ProductTier | 'none';