[8.x] [Cloud Security] [CDR] Handle grouping fields with missing mapping (#195702)

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts

@@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common';
 import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
 import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
+import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils';
 
@@ -39,6 +40,20 @@ interface FindingsAggs {
   count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
 }
 
+const getRuntimeMappingsFromSort = (sort: string[][]) => {
+  return sort.reduce((acc, [field]) => {
+    // TODO: Add proper type for all fields available in the field selector
+    const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword';
+
+    return {
+      ...acc,
+      [field]: {
+        type,
+      },
+    };
+  }, {});
+};
+
 export const getFindingsQuery = (
   { query, sort }: UseFindingsOptions,
   rulesStates: CspBenchmarkRulesStates,
@@ -49,6 +64,7 @@ export const getFindingsQuery = (
   return {
     index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
     sort: getMultiFieldsSort(sort),
+    runtime_mappings: getRuntimeMappingsFromSort(sort),
     size: MAX_FINDINGS_TO_LOAD,
     aggs: getFindingsCountAggQuery(),
     ignore_unavailable: true,

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx

@@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
+/**
+ * Get runtime mappings for the given group field
+ * Some fields require additional runtime mappings to aggregate additional information
+ * Fallback to keyword type to support custom fields grouping
+ */
+const getRuntimeMappingsByGroupField = (
+  field: string
+): Record<string, { type: 'keyword' }> | undefined => {
+  switch (field) {
+    case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: {
+          type: 'keyword',
+        },
+        'resource.id': {
+          type: 'keyword',
+        },
+        'resource.sub_type': {
+          type: 'keyword',
+        },
+        'resource.type': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.RULE_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.version': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.name': {
+          type: 'keyword',
+        },
+        'rule.benchmark.id': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.name': {
+          type: 'keyword',
+        },
+        'rule.benchmark.id': {
+          type: 'keyword',
+        },
+      };
+    default:
+      return {
+        [field]: {
+          type: 'keyword',
+        },
+      };
+  }
+};
+
 /**
  * Type Guard for checking if the given source is a FindingsRootGroupingAggregation
  */
@@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
+    runtimeMappings: {
+      ...getRuntimeMappingsByGroupField(currentSelectedGroup),
+      'result.evaluation': {
+        type: 'keyword',
+      },
+    },
     rootAggregations: [
       {
         failedFindings: {

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx

@@ -23,6 +23,7 @@ import {
 } from '@kbn/cloud-security-posture-common';
 import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
 import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
+import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
 import { VULNERABILITY_FIELDS } from '../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script';
@@ -52,13 +53,33 @@ const getMultiFieldsSort = (sort: string[][]) => {
   });
 };
 
+const getRuntimeMappingsFromSort = (sort: string[][]) => {
+  return sort.reduce((acc, [field]) => {
+    // TODO: Add proper type for all fields available in the field selector
+    const type: RuntimePrimitiveTypes =
+      field === VULNERABILITY_FIELDS.SCORE_BASE
+        ? 'double'
+        : field === '@timestamp'
+        ? 'date'
+        : 'keyword';
+
+    return {
+      ...acc,
+      [field]: {
+        type,
+      },
+    };
+  }, {});
+};
+
 export const getVulnerabilitiesQuery = (
   { query, sort }: VulnerabilitiesQuery,
   pageParam: number
 ) => ({
   index: CDR_VULNERABILITIES_INDEX_PATTERN,
   ignore_unavailable: true,
   sort: getMultiFieldsSort(sort),
+  runtime_mappings: getRuntimeMappingsFromSort(sort),
   size: MAX_FINDINGS_TO_LOAD,
   query: {
     ...query,

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx

@@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
+/**
+ * Get runtime mappings for the given group field
+ * Some fields require additional runtime mappings to aggregate additional information
+ * Fallback to keyword type to support custom fields grouping
+ */
+const getRuntimeMappingsByGroupField = (
+  field: string
+): Record<string, { type: 'keyword' }> | undefined => {
+  switch (field) {
+    case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: {
+          type: 'keyword',
+        },
+      };
+    case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.RESOURCE_ID]: {
+          type: 'keyword',
+        },
+      };
+    case VULNERABILITY_GROUPING_OPTIONS.CVE:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.CVE]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.DESCRIPTION]: {
+          type: 'keyword',
+        },
+      };
+    default:
+      return {
+        [field]: {
+          type: 'keyword',
+        },
+      };
+  }
+};
+
 /**
  * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation
  */
@@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
+    runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup),
   });
 
   const { data, isFetching } = useGroupedVulnerabilities({

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts

@@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) =
       type: 'string',
       order: direction,
       script: {
-        source: `doc["${field}"].value.toLowerCase()`,
+        source: `
+          if (doc.containsKey('${field}') && !doc['${field}'].empty) {
+            return doc['${field}'].value.toLowerCase();
+          } else {
+            return "";
+          }
+        `,
         lang: 'painless',
       },
     },