[Cloud Security] [CDR] Handle grouping fields with missing mapping (e…

…lastic#195702)

## Summary

This PR fixes elastic/security-team#10632 by
adding runtime mapping support for fields that are missing in mapping,
this is useful when querying a DataView that points to multiple indices
where the mapping is not guaranteed to exist as it's the case with CDR
that adds supports to Third Party data.

Also added runtime mapping to sorted fields, as it's not guaranteed that
all fields shown on the table have mapped fields.

(cherry picked from commit e53e545)

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts

@@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common';
 import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
 import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
+import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils';
 
@@ -39,6 +40,20 @@ interface FindingsAggs {
   count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
 }
 
+const getRuntimeMappingsFromSort = (sort: string[][]) => {
+  return sort.reduce((acc, [field]) => {
+    // TODO: Add proper type for all fields available in the field selector
+    const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword';
+
+    return {
+      ...acc,
+      [field]: {
+        type,
+      },
+    };
+  }, {});
+};
+
 export const getFindingsQuery = (
   { query, sort }: UseFindingsOptions,
   rulesStates: CspBenchmarkRulesStates,
@@ -49,6 +64,7 @@ export const getFindingsQuery = (
   return {
     index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
     sort: getMultiFieldsSort(sort),
+    runtime_mappings: getRuntimeMappingsFromSort(sort),
     size: MAX_FINDINGS_TO_LOAD,
     aggs: getFindingsCountAggQuery(),
     ignore_unavailable: true,

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx

@@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
+/**
+ * Get runtime mappings for the given group field
+ * Some fields require additional runtime mappings to aggregate additional information
+ * Fallback to keyword type to support custom fields grouping
+ */
+const getRuntimeMappingsByGroupField = (
+  field: string
+): Record<string, { type: 'keyword' }> | undefined => {
+  switch (field) {
+    case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: {
+          type: 'keyword',
+        },
+        'resource.id': {
+          type: 'keyword',
+        },
+        'resource.sub_type': {
+          type: 'keyword',
+        },
+        'resource.type': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.RULE_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.version': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.name': {
+          type: 'keyword',
+        },
+        'rule.benchmark.id': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.name': {
+          type: 'keyword',
+        },
+        'rule.benchmark.id': {
+          type: 'keyword',
+        },
+      };
+    default:
+      return {
+        [field]: {
+          type: 'keyword',
+        },
+      };
+  }
+};
+
 /**
  * Type Guard for checking if the given source is a FindingsRootGroupingAggregation
  */
@@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
+    runtimeMappings: {
+      ...getRuntimeMappingsByGroupField(currentSelectedGroup),
+      'result.evaluation': {
+        type: 'keyword',
+      },
+    },
     rootAggregations: [
       {
         failedFindings: {

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx

@@ -23,6 +23,7 @@ import {
 } from '@kbn/cloud-security-posture-common';
 import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
 import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
+import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
 import { VULNERABILITY_FIELDS } from '../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script';
@@ -52,13 +53,33 @@ const getMultiFieldsSort = (sort: string[][]) => {
   });
 };
 
+const getRuntimeMappingsFromSort = (sort: string[][]) => {
+  return sort.reduce((acc, [field]) => {
+    // TODO: Add proper type for all fields available in the field selector
+    const type: RuntimePrimitiveTypes =
+      field === VULNERABILITY_FIELDS.SCORE_BASE
+        ? 'double'
+        : field === '@timestamp'
+        ? 'date'
+        : 'keyword';
+
+    return {
+      ...acc,
+      [field]: {
+        type,
+      },
+    };
+  }, {});
+};
+
 export const getVulnerabilitiesQuery = (
   { query, sort }: VulnerabilitiesQuery,
   pageParam: number
 ) => ({
   index: CDR_VULNERABILITIES_INDEX_PATTERN,
   ignore_unavailable: true,
   sort: getMultiFieldsSort(sort),
+  runtime_mappings: getRuntimeMappingsFromSort(sort),
   size: MAX_FINDINGS_TO_LOAD,
   query: {
     ...query,

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx

@@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
+/**
+ * Get runtime mappings for the given group field
+ * Some fields require additional runtime mappings to aggregate additional information
+ * Fallback to keyword type to support custom fields grouping
+ */
+const getRuntimeMappingsByGroupField = (
+  field: string
+): Record<string, { type: 'keyword' }> | undefined => {
+  switch (field) {
+    case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: {
+          type: 'keyword',
+        },
+      };
+    case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.RESOURCE_ID]: {
+          type: 'keyword',
+        },
+      };
+    case VULNERABILITY_GROUPING_OPTIONS.CVE:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.CVE]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.DESCRIPTION]: {
+          type: 'keyword',
+        },
+      };
+    default:
+      return {
+        [field]: {
+          type: 'keyword',
+        },
+      };
+  }
+};
+
 /**
  * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation
  */
@@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
+    runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup),
   });
 
   const { data, isFetching } = useGroupedVulnerabilities({

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts

@@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) =
       type: 'string',
       order: direction,
       script: {
-        source: `doc["${field}"].value.toLowerCase()`,
+        source: `
+          if (doc.containsKey('${field}') && !doc['${field}'].empty) {
+            return doc['${field}'].value.toLowerCase();
+          } else {
+            return "";
+          }
+        `,
         lang: 'painless',
       },
     },