add logger and tests

x-pack/plugins/cloud_security_posture/server/lib/mapping_field_util.ts

@@ -5,22 +5,14 @@
  * 2.0.
  */
 
-export const DELIMITER = ';';
 export const MAPPING_VERSION_DELIMITER = '_';
-export const DOC_FIELD_VERSION_DELIMITER = '.';
 
 export const toBenchmarkDocFieldKey = (benchmarkId: string, benchmarkVersion: string) => {
   if (benchmarkVersion.includes(MAPPING_VERSION_DELIMITER))
-    return `${benchmarkId}${DELIMITER}${benchmarkVersion.replaceAll(
-      `${MAPPING_VERSION_DELIMITER}`,
-      DOC_FIELD_VERSION_DELIMITER
-    )}`;
-  return `${benchmarkId}${DELIMITER}${benchmarkVersion}`;
+    return `${benchmarkId};${benchmarkVersion.replaceAll('_', '.')}`;
+  return `${benchmarkId};${benchmarkVersion}`;
 };
 
 export const toBenchmarkMappingFieldKey = (benchmarkVersion: string) => {
-  return `${benchmarkVersion.replace(
-    `/${DOC_FIELD_VERSION_DELIMITER}/g`,
-    MAPPING_VERSION_DELIMITER
-  )}`;
+  return `${benchmarkVersion.replaceAll('.', '_')}`;
 };

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/compliance_dashboard.ts

@@ -38,20 +38,22 @@ const getClustersTrends = (clustersWithoutTrends: ClusterWithoutTrend[], trends:
     })),
   }));
 
-const getBenchmarksTrends = (benchmarksWithoutTrends: BenchmarkWithoutTrend[], trends: Trends) =>
-  benchmarksWithoutTrends.map((benchmark) => ({
+const getBenchmarksTrends = (benchmarksWithoutTrends: BenchmarkWithoutTrend[], trends: Trends) => {
+  return benchmarksWithoutTrends.map((benchmark) => ({
     ...benchmark,
     trend: trends.map(({ timestamp, benchmarks: benchmarksTrendData }) => {
       const benchmarkIdVersion = toBenchmarkDocFieldKey(
         benchmark.meta.benchmarkId,
-        benchmark.meta.benchmarkName
+        benchmark.meta.benchmarkVersion
       );
+
       return {
         timestamp,
         ...benchmarksTrendData[benchmarkIdVersion],
       };
     }),
   }));
+};
 
 const getSummaryTrend = (trends: Trends) =>
   trends.map(({ timestamp, summary }) => ({ timestamp, ...summary }));
@@ -73,6 +75,7 @@ export const defineGetComplianceDashboardRoute = (router: CspRouter) =>
       },
       async (context, request, response) => {
         const cspContext = await context.csp;
+        const logger = cspContext.logger;
 
         try {
           const esClient = cspContext.esClient.asCurrentUser;
@@ -101,17 +104,17 @@ export const defineGetComplianceDashboardRoute = (router: CspRouter) =>
             benchmarksWithoutTrends,
             trends,
           ] = await Promise.all([
-            getStats(esClient, query, pitId, runtimeMappings),
-            getGroupedFindingsEvaluation(esClient, query, pitId, runtimeMappings),
-            getClusters(esClient, query, pitId, runtimeMappings),
-            getBenchmarks(esClient, query, pitId, runtimeMappings),
-            getTrends(esClient, policyTemplate),
+            getStats(logger, esClient, query, pitId, runtimeMappings),
+            getGroupedFindingsEvaluation(logger, esClient, query, pitId, runtimeMappings),
+            getClusters(logger, esClient, query, pitId, runtimeMappings),
+            getBenchmarks(logger, esClient, query, pitId, runtimeMappings),
+            getTrends(logger, esClient, policyTemplate),
           ]);
 
           // Try closing the PIT, if it fails we can safely ignore the error since it closes itself after the keep alive
           //   ends. Not waiting on the promise returned from the `closePointInTime` call to avoid delaying the request
           esClient.closePointInTime({ id: pitId }).catch((err) => {
-            cspContext.logger.warn(`Could not close PIT for stats endpoint: ${err}`);
+            logger.warn(`Could not close PIT for stats endpoint: ${err}`);
           });
 
           const clusters = getClustersTrends(clustersWithoutTrends, trends);
@@ -131,7 +134,8 @@ export const defineGetComplianceDashboardRoute = (router: CspRouter) =>
           });
         } catch (err) {
           const error = transformError(err);
-          cspContext.logger.error(`Error while fetching CSP stats: ${err}`);
+          logger.error(`Error while fetching CSP stats: ${err}`);
+          logger.error(err.stack);
 
           return response.customError({
             body: { message: error.message },

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_benchmarks.ts

@@ -11,6 +11,7 @@ import type {
   QueryDslQueryContainer,
   SearchRequest,
 } from '@elastic/elasticsearch/lib/api/types';
+import type { Logger } from '@kbn/core/server';
 import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import type { BenchmarkData } from '../../../common/types';
 import {
@@ -124,17 +125,24 @@ export const getBenchmarksFromAggs = (benchmarks: BenchmarkBucket[]) => {
 };
 
 export const getBenchmarks = async (
+  logger: Logger,
   esClient: ElasticsearchClient,
   query: QueryDslQueryContainer,
   pitId: string,
   runtimeMappings: MappingRuntimeFields
 ): Promise<BenchmarkWithoutTrend[]> => {
-  const queryResult = await esClient.search<unknown, BenchmarkQueryResult>(
-    getBenchmarksQuery(query, pitId, runtimeMappings)
-  );
+  try {
+    const queryResult = await esClient.search<unknown, BenchmarkQueryResult>(
+      getBenchmarksQuery(query, pitId, runtimeMappings)
+    );
 
-  const benchmarks = queryResult.aggregations?.aggs_by_benchmark.buckets;
-  if (!Array.isArray(benchmarks)) throw new Error('missing aggs by benchmark id');
+    const benchmarks = queryResult.aggregations?.aggs_by_benchmark.buckets;
+    if (!Array.isArray(benchmarks)) throw new Error('missing aggs by benchmark id');
 
-  return getBenchmarksFromAggs(benchmarks);
+    return getBenchmarksFromAggs(benchmarks);
+  } catch (err) {
+    logger.error(`Failed to fetch benchmark stats ${err.message}`);
+    logger.error(err);
+    throw err;
+  }
 };

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_clusters.ts

@@ -13,6 +13,7 @@ import type {
   AggregationsTopHitsAggregate,
   SearchHit,
 } from '@elastic/elasticsearch/lib/api/types';
+import type { Logger } from '@kbn/core/server';
 import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import { CspFinding } from '../../../common/schemas/csp_finding';
 import type { Cluster } from '../../../common/types';
@@ -109,17 +110,24 @@ export const getClustersFromAggs = (clusters: ClusterBucket[]): ClusterWithoutTr
   });
 
 export const getClusters = async (
+  logger: Logger,
   esClient: ElasticsearchClient,
   query: QueryDslQueryContainer,
   pitId: string,
   runtimeMappings: MappingRuntimeFields
 ): Promise<ClusterWithoutTrend[]> => {
-  const queryResult = await esClient.search<unknown, ClustersQueryResult>(
-    getClustersQuery(query, pitId, runtimeMappings)
-  );
+  try {
+    const queryResult = await esClient.search<unknown, ClustersQueryResult>(
+      getClustersQuery(query, pitId, runtimeMappings)
+    );
 
-  const clusters = queryResult.aggregations?.aggs_by_asset_identifier.buckets;
-  if (!Array.isArray(clusters)) throw new Error('missing aggs by cluster id');
+    const clusters = queryResult.aggregations?.aggs_by_asset_identifier.buckets;
+    if (!Array.isArray(clusters)) throw new Error('missing aggs by cluster id');
 
-  return getClustersFromAggs(clusters);
+    return getClustersFromAggs(clusters);
+  } catch (err) {
+    logger.error(`Failed to fetch cluster stats ${err.message}`);
+    logger.error(err);
+    throw err;
+  }
 };

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_grouped_findings_evaluation.ts

@@ -11,6 +11,7 @@ import type {
   QueryDslQueryContainer,
   SearchRequest,
 } from '@elastic/elasticsearch/lib/api/types';
+import type { Logger } from '@kbn/core/server';
 import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import { calculatePostureScore } from '../../../common/utils/helpers';
 import type { ComplianceDashboardData } from '../../../common/types';
@@ -118,19 +119,26 @@ export const getFailedFindingsFromAggs = (
   });
 
 export const getGroupedFindingsEvaluation = async (
+  logger: Logger,
   esClient: ElasticsearchClient,
   query: QueryDslQueryContainer,
   pitId: string,
   runtimeMappings: MappingRuntimeFields
 ): Promise<ComplianceDashboardData['groupedFindingsEvaluation']> => {
-  const resourceTypesQueryResult = await esClient.search<unknown, FailedFindingsQueryResult>(
-    getRisksEsQuery(query, pitId, runtimeMappings)
-  );
+  try {
+    const resourceTypesQueryResult = await esClient.search<unknown, FailedFindingsQueryResult>(
+      getRisksEsQuery(query, pitId, runtimeMappings)
+    );
 
-  const ruleSections = resourceTypesQueryResult.aggregations?.aggs_by_resource_type.buckets;
-  if (!Array.isArray(ruleSections)) {
-    return [];
-  }
+    const ruleSections = resourceTypesQueryResult.aggregations?.aggs_by_resource_type.buckets;
+    if (!Array.isArray(ruleSections)) {
+      return [];
+    }
 
-  return getFailedFindingsFromAggs(ruleSections);
+    return getFailedFindingsFromAggs(ruleSections);
+  } catch (err) {
+    logger.error(`Failed to fetch findings stats ${err.message}`);
+    logger.error(err);
+    throw err;
+  }
 };

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_stats.ts

@@ -8,6 +8,7 @@
 import { ElasticsearchClient } from '@kbn/core/server';
 import type { QueryDslQueryContainer, SearchRequest } from '@elastic/elasticsearch/lib/api/types';
 import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
+import type { Logger } from '@kbn/core/server';
 import { calculatePostureScore } from '../../../common/utils/helpers';
 import type { ComplianceDashboardData } from '../../../common/types';
 
@@ -78,17 +79,24 @@ export const getStatsFromFindingsEvaluationsAggs = (
 };
 
 export const getStats = async (
+  logger: Logger,
   esClient: ElasticsearchClient,
   query: QueryDslQueryContainer,
   pitId: string,
   runtimeMappings: MappingRuntimeFields
 ): Promise<ComplianceDashboardData['stats']> => {
-  const evaluationsQueryResult = await esClient.search<unknown, FindingsEvaluationsQueryResult>(
-    getEvaluationsQuery(query, pitId, runtimeMappings)
-  );
+  try {
+    const evaluationsQueryResult = await esClient.search<unknown, FindingsEvaluationsQueryResult>(
+      getEvaluationsQuery(query, pitId, runtimeMappings)
+    );
 
-  const findingsEvaluations = evaluationsQueryResult.aggregations;
-  if (!findingsEvaluations) throw new Error('missing findings evaluations');
+    const findingsEvaluations = evaluationsQueryResult.aggregations;
+    if (!findingsEvaluations) throw new Error('missing findings evaluations');
 
-  return getStatsFromFindingsEvaluationsAggs(findingsEvaluations);
+    return getStatsFromFindingsEvaluationsAggs(findingsEvaluations);
+  } catch (err) {
+    logger.error(`Failed to fetch stats ${err.message}`);
+    logger.error(err);
+    throw err;
+  }
 };

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_trends.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { getTrendsFromQueryResult, ScoreTrendDoc } from './get_trends';
+import { formatTrends, ScoreTrendDoc } from './get_trends';
 
 const trendDocs: ScoreTrendDoc[] = [
   {
@@ -75,7 +75,7 @@ const trendDocs: ScoreTrendDoc[] = [
 
 describe('getTrendsFromQueryResult', () => {
   it('should return value matching Trends type definition, in descending order, and with postureScore', async () => {
-    const trends = getTrendsFromQueryResult(trendDocs);
+    const trends = formatTrends(trendDocs);
     expect(trends).toEqual([
       {
         timestamp: '2022-04-06T15:30:00Z',

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_trends.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ElasticsearchClient } from '@kbn/core/server';
+import { ElasticsearchClient, Logger } from '@kbn/core/server';
 import { calculatePostureScore } from '../../../common/utils/helpers';
 import { BENCHMARK_SCORE_INDEX_DEFAULT_NS } from '../../../common/constants';
 import type { PosturePolicyTemplate, Stats } from '../../../common/types';
@@ -64,7 +64,7 @@ export const getTrendsQuery = (policyTemplate: PosturePolicyTemplate) => ({
   },
 });
 
-export const getTrendsFromQueryResult = (scoreTrendDocs: ScoreTrendDoc[]): Trends => {
+export const formatTrends = (scoreTrendDocs: ScoreTrendDoc[]): Trends => {
   return scoreTrendDocs.map((data) => {
     return {
       timestamp: data['@timestamp'],
@@ -111,17 +111,24 @@ export const getTrendsFromQueryResult = (scoreTrendDocs: ScoreTrendDoc[]): Trend
 };
 
 export const getTrends = async (
+  logger: Logger,
   esClient: ElasticsearchClient,
   policyTemplate: PosturePolicyTemplate
 ): Promise<Trends> => {
-  const trendsQueryResult = await esClient.search<ScoreTrendDoc>(getTrendsQuery(policyTemplate));
+  try {
+    const trendsQueryResult = await esClient.search<ScoreTrendDoc>(getTrendsQuery(policyTemplate));
 
-  if (!trendsQueryResult.hits.hits) throw new Error('missing trend results from score index');
+    if (!trendsQueryResult.hits.hits) throw new Error('missing trend results from score index');
 
-  const scoreTrendDocs = trendsQueryResult.hits.hits.map((hit) => {
-    if (!hit._source) throw new Error('missing _source data for one or more of trend results');
-    return hit._source;
-  });
+    const scoreTrendDocs = trendsQueryResult.hits.hits.map((hit) => {
+      if (!hit._source) throw new Error('missing _source data for one or more of trend results');
+      return hit._source;
+    });
 
-  return getTrendsFromQueryResult(scoreTrendDocs);
+    return formatTrends(scoreTrendDocs);
+  } catch (err) {
+    logger.error(`Failed to fetch trendlines data ${err.message}`);
+    logger.error(err);
+    throw err;
+  }
 };

x-pack/test/cloud_security_posture_api/config.ts

@@ -17,6 +17,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     testFiles: [
       require.resolve('./telemetry/telemetry.ts'),
       require.resolve('./routes/vulnerabilities_dashboard.ts'),
+      require.resolve('./routes/stats.ts'),
     ],
     junit: {
       reportName: 'X-Pack Cloud Security Posture API Tests',