Skip to content

Commit

Permalink
report all assets details
Browse files Browse the repository at this point in the history
  • Loading branch information
@CohenIdo
CohenIdo committed Jun 23, 2024
1 parent 6d39b8a commit 279db5e
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 97 deletions.
9 changes: 9 additions & 0 deletions x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ export const cloudSecurityMetringCallback = async ({
lastSuccessfulReport,
config,
}: MeteringCallbackInput): Promise<MeteringCallBackResponse> => {
const projectHasCloudProductLine = config.productTypes.some(
(product) => product.product_line === ProductLine.cloud
);

if (!projectHasCloudProductLine) {
logger.info('No cloud product line found in the project');
return { records: [] };
}

const projectId = cloudSetup?.serverless?.projectId || 'missing_project_id';

const tier: Tier = getCloudProductTier(config, logger);
Expand Down
196 changes: 101 additions & 95 deletions ...lugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,77 +15,112 @@ import {
CSPM,
KSPM,
METERING_CONFIGS,
THRESHOLD_MINUTES,
BILLABLE_ASSETS_CONFIG,
} from './constants';
import type { Tier, UsageRecord } from '../types';
import type {
CloudSecurityMeteringCallbackInput,
CloudSecuritySolutions,
AssetCountAggregation,
ResourceSubtypeAggregationBucket,
} from './types';

export interface ResourceSubtypeCounter {
[key: string]: {
doc_count: number;
unique_assets: number;
};
}

export const getUsageRecords = (
assetCountAggregations: AssetCountAggregation[],
assetCountAggregation: AssetCountAggregation,
cloudSecuritySolution: CloudSecuritySolutions,
taskId: string,
tier: Tier,
projectId: string,
periodSeconds: number,
logger: Logger
): UsageRecord[] => {
const usageRecords = assetCountAggregations.map((assetCountAggregation) => {
const assetCount = assetCountAggregation.unique_assets.value;

if (assetCount > AGGREGATION_PRECISION_THRESHOLD) {
logger.warn(
`The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.`
);
}

const minTimestamp = new Date(
assetCountAggregation.min_timestamp.value_as_string
).toISOString();

const creationTimestamp = new Date();
const minutes = creationTimestamp.getMinutes();
if (minutes >= 30) {
creationTimestamp.setMinutes(30, 0, 0);
} else {
creationTimestamp.setMinutes(0, 0, 0);
}
const roundedCreationTimestamp = creationTimestamp.toISOString();

const usageRecord: UsageRecord = {
id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`,
usage_timestamp: minTimestamp,
creation_timestamp: creationTimestamp.toISOString(),
usage: {
type: CLOUD_SECURITY_TASK_TYPE,
sub_type: cloudSecuritySolution,
quantity: assetCount,
period_seconds: periodSeconds,
},
source: {
id: taskId,
instance_group_id: projectId,
metadata: { tier },
},
};
): UsageRecord => {
const resourceSubtypeCounter = assetCountAggregation.resource_sub_type.buckets.reduce(
(resourceMap, item) => {
resourceMap[item.key] = {
doc_count: item.doc_count,
unique_assets: item.unique_assets.value,
};
return resourceMap;
},
{} as ResourceSubtypeCounter
);

const resourceSubtypeBuckets: ResourceSubtypeAggregationBucket[] =
assetCountAggregation.resource_sub_type.buckets;

let assetCount;

if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) {
const billableAssets = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution].values;
assetCount = resourceSubtypeBuckets
.filter((bucket) => billableAssets.includes(bucket.key))
.reduce((acc, bucket) => acc + bucket.unique_assets.value, 0);
} else {
assetCount = resourceSubtypeBuckets.reduce(
(acc, bucket) => acc + bucket.unique_assets.value,
0
);
}

return usageRecord;
});
return usageRecords;
if (assetCount > AGGREGATION_PRECISION_THRESHOLD) {
logger.warn(
`The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.`
);
}

const minTimestamp = new Date(assetCountAggregation.min_timestamp.value_as_string).toISOString();

const creationTimestamp = new Date();
const minutes = creationTimestamp.getMinutes();
if (minutes >= 30) {
creationTimestamp.setMinutes(30, 0, 0);
} else {
creationTimestamp.setMinutes(0, 0, 0);
}
const roundedCreationTimestamp = creationTimestamp.toISOString();

const usageRecord: UsageRecord = {
id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${roundedCreationTimestamp}`,
usage_timestamp: minTimestamp,
creation_timestamp: creationTimestamp.toISOString(),
usage: {
type: CLOUD_SECURITY_TASK_TYPE,
sub_type: cloudSecuritySolution,
quantity: assetCount,
period_seconds: periodSeconds,
},
source: {
id: taskId,
instance_group_id: projectId,
metadata: { tier, resourceSubtypeCounter },
},
};

return usageRecord;
};

export const getAggregationByCloudSecuritySolution = (
cloudSecuritySolution: CloudSecuritySolutions
) => {
return {
unique_assets: {
cardinality: {
field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier,
precision_threshold: AGGREGATION_PRECISION_THRESHOLD,
resource_sub_type: {
terms: {
field: `resource.sub_type`,
},
aggs: {
unique_assets: {
cardinality: {
field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier,
precision_threshold: AGGREGATION_PRECISION_THRESHOLD,
},
},
},
},
min_timestamp: {
Expand All @@ -97,8 +132,7 @@ export const getAggregationByCloudSecuritySolution = (
};

export const getSearchQueryByCloudSecuritySolution = (
cloudSecuritySolution: CloudSecuritySolutions,
searchFrom: Date
cloudSecuritySolution: CloudSecuritySolutions
) => {
const mustFilters = [];

Expand All @@ -117,20 +151,11 @@ export const getSearchQueryByCloudSecuritySolution = (
}

if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) {
const billableAssetsConfig = BILLABLE_ASSETS_CONFIG[cloudSecuritySolution];

mustFilters.push({
term: {
'rule.benchmark.posture_type': cloudSecuritySolution,
},
});

// filter in only billable assets
mustFilters.push({
terms: {
[billableAssetsConfig.filter_attribute]: billableAssetsConfig.values,
},
});
}

return {
Expand All @@ -141,10 +166,9 @@ export const getSearchQueryByCloudSecuritySolution = (
};

export const getAssetAggQueryByCloudSecuritySolution = (
cloudSecuritySolution: CloudSecuritySolutions,
searchFrom: Date
cloudSecuritySolution: CloudSecuritySolutions
) => {
const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom);
const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution);
const aggs = getAggregationByCloudSecuritySolution(cloudSecuritySolution);

return {
Expand All @@ -157,51 +181,34 @@ export const getAssetAggQueryByCloudSecuritySolution = (

export const getAssetAggByCloudSecuritySolution = async (
esClient: ElasticsearchClient,
cloudSecuritySolution: CloudSecuritySolutions,
searchFrom: Date
): Promise<AssetCountAggregation[]> => {
const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom);
cloudSecuritySolution: CloudSecuritySolutions
): Promise<AssetCountAggregation | undefined> => {
const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution);

const response = await esClient.search<unknown, AssetCountAggregation>(assetsAggQuery);
if (!response.aggregations) return [];

return [response.aggregations];
if (!response.aggregations) return;

return response.aggregations;
};

const indexHasDataInDateRange = async (
esClient: ElasticsearchClient,
cloudSecuritySolution: CloudSecuritySolutions,
searchFrom: Date
cloudSecuritySolution: CloudSecuritySolutions
) => {
const response = await esClient.search(
{
index: METERING_CONFIGS[cloudSecuritySolution].index,
size: 1,
_source: false,
query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom),
query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution),
},
{ ignore: [404] }
);

return response.hits?.hits.length > 0;
};

const getSearchStartDate = (lastSuccessfulReport: Date): Date => {
const initialDate = new Date();
const thresholdDate = new Date(initialDate.getTime() - THRESHOLD_MINUTES * 60 * 1000);

if (lastSuccessfulReport) {
const lastSuccessfulReportDate = new Date(lastSuccessfulReport);

const searchFrom =
lastSuccessfulReport && lastSuccessfulReportDate > thresholdDate
? lastSuccessfulReportDate
: thresholdDate;
return searchFrom;
}
return thresholdDate;
};

export const getCloudSecurityUsageRecord = async ({
esClient,
projectId,
Expand All @@ -212,21 +219,20 @@ export const getCloudSecurityUsageRecord = async ({
logger,
}: CloudSecurityMeteringCallbackInput): Promise<UsageRecord[] | undefined> => {
try {
const searchFrom = getSearchStartDate(lastSuccessfulReport);

if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution, searchFrom))) return;
if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution))) return;

// const periodSeconds = Math.floor((new Date().getTime() - searchFrom.getTime()) / 1000);
const periodSeconds = 1800; // Workaround to prevent overbilling by charging for a constant time window. The issue should be resolved in https://github.com/elastic/security-team/issues/9424.

const assetCountAggregations = await getAssetAggByCloudSecuritySolution(
const assetCountAggregation = await getAssetAggByCloudSecuritySolution(
esClient,
cloudSecuritySolution,
searchFrom
cloudSecuritySolution
);

if (!assetCountAggregation) return [];

const usageRecords = await getUsageRecords(
assetCountAggregations,
assetCountAggregation,
cloudSecuritySolution,
taskId,
tier,
Expand All @@ -235,7 +241,7 @@ export const getCloudSecurityUsageRecord = async ({
logger
);

return usageRecords;
return [usageRecords];
} catch (err) {
logger.error(`Failed to fetch ${cloudSecuritySolution} metering data ${err}`);
}
Expand Down
13 changes: 11 additions & 2 deletions x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,20 @@ export interface CloudDefendAssetCountAggregation {
export interface AssetCountAggregationBucket {
buckets: AssetCountAggregation[];
}

export interface ResourceSubtypeAggregationBucket {
key: string;
doc_count: number;
unique_assets: {
value: number;
};
}

export interface AssetCountAggregation {
key_as_string: string;
min_timestamp: MinTimestamp;
unique_assets: {
value: number;
resource_sub_type: {
buckets: ResourceSubtypeAggregationBucket[];
};
}

Expand Down
2 changes: 2 additions & 0 deletions x-pack/plugins/security_solution_serverless/server/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import type { ServerlessPluginSetup } from '@kbn/serverless/server';
import type { ProductTier } from '../common/product';

import type { ServerlessSecurityConfig } from './config';
import { ResourceSubtypeCounter } from './cloud_security/cloud_security_metering_task';

// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface SecuritySolutionServerlessPluginSetup {}
Expand Down Expand Up @@ -74,6 +75,7 @@ export interface UsageSource {

export interface UsageSourceMetadata {
tier?: Tier;
resourceSubtypeCounter?: ResourceSubtypeCounter;
}

export type Tier = ProductTier | 'none';
Expand Down

0 comments on commit 279db5e

Please sign in to comment.