[8.x] [Cloud Security] [CDR] Handle grouping fields with missing mapp…

…ing (#195702) (#195792)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Cloud Security] [CDR] Handle grouping fields with missing mapping
(#195702)](#195702)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Paulo
Silva","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-10T14:52:49Z","message":"[Cloud
Security] [CDR] Handle grouping fields with missing mapping
(#195702)\n\n## Summary\r\n\r\nThis PR fixes
elastic/security-team#10632 by\r\nadding
runtime mapping support for fields that are missing in mapping,\r\nthis
is useful when querying a DataView that points to multiple
indices\r\nwhere the mapping is not guaranteed to exist as it's the case
with CDR\r\nthat adds supports to Third Party data.\r\n\r\nAlso added
runtime mapping to sorted fields, as it's not guaranteed that\r\nall
fields shown on the table have mapped
fields.","sha":"e53e54550f9ab9ce2db83ec56a5c704a96f37355","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Cloud
Security","v8.16.0","backport:version"],"title":"[Cloud Security] [CDR]
Handle grouping fields with missing
mapping","number":195702,"url":"https://github.com/elastic/kibana/pull/195702","mergeCommit":{"message":"[Cloud
Security] [CDR] Handle grouping fields with missing mapping
(#195702)\n\n## Summary\r\n\r\nThis PR fixes
elastic/security-team#10632 by\r\nadding
runtime mapping support for fields that are missing in mapping,\r\nthis
is useful when querying a DataView that points to multiple
indices\r\nwhere the mapping is not guaranteed to exist as it's the case
with CDR\r\nthat adds supports to Third Party data.\r\n\r\nAlso added
runtime mapping to sorted fields, as it's not guaranteed that\r\nall
fields shown on the table have mapped
fields.","sha":"e53e54550f9ab9ce2db83ec56a5c704a96f37355"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195702","number":195702,"mergeCommit":{"message":"[Cloud
Security] [CDR] Handle grouping fields with missing mapping
(#195702)\n\n## Summary\r\n\r\nThis PR fixes
elastic/security-team#10632 by\r\nadding
runtime mapping support for fields that are missing in mapping,\r\nthis
is useful when querying a DataView that points to multiple
indices\r\nwhere the mapping is not guaranteed to exist as it's the case
with CDR\r\nthat adds supports to Third Party data.\r\n\r\nAlso added
runtime mapping to sorted fields, as it's not guaranteed that\r\nall
fields shown on the table have mapped
fields.","sha":"e53e54550f9ab9ce2db83ec56a5c704a96f37355"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Paulo Silva <[email protected]>

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts

@@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common';
 import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
 import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
+import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils';
 
@@ -39,6 +40,20 @@ interface FindingsAggs {
   count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
 }
 
+const getRuntimeMappingsFromSort = (sort: string[][]) => {
+  return sort.reduce((acc, [field]) => {
+    // TODO: Add proper type for all fields available in the field selector
+    const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword';
+
+    return {
+      ...acc,
+      [field]: {
+        type,
+      },
+    };
+  }, {});
+};
+
 export const getFindingsQuery = (
   { query, sort }: UseFindingsOptions,
   rulesStates: CspBenchmarkRulesStates,
@@ -49,6 +64,7 @@ export const getFindingsQuery = (
   return {
     index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
     sort: getMultiFieldsSort(sort),
+    runtime_mappings: getRuntimeMappingsFromSort(sort),
     size: MAX_FINDINGS_TO_LOAD,
     aggs: getFindingsCountAggQuery(),
     ignore_unavailable: true,

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx

@@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
+/**
+ * Get runtime mappings for the given group field
+ * Some fields require additional runtime mappings to aggregate additional information
+ * Fallback to keyword type to support custom fields grouping
+ */
+const getRuntimeMappingsByGroupField = (
+  field: string
+): Record<string, { type: 'keyword' }> | undefined => {
+  switch (field) {
+    case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: {
+          type: 'keyword',
+        },
+        'resource.id': {
+          type: 'keyword',
+        },
+        'resource.sub_type': {
+          type: 'keyword',
+        },
+        'resource.type': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.RULE_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.version': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.name': {
+          type: 'keyword',
+        },
+        'rule.benchmark.id': {
+          type: 'keyword',
+        },
+      };
+    case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME:
+      return {
+        [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: {
+          type: 'keyword',
+        },
+        'rule.benchmark.name': {
+          type: 'keyword',
+        },
+        'rule.benchmark.id': {
+          type: 'keyword',
+        },
+      };
+    default:
+      return {
+        [field]: {
+          type: 'keyword',
+        },
+      };
+  }
+};
+
 /**
  * Type Guard for checking if the given source is a FindingsRootGroupingAggregation
  */
@@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
+    runtimeMappings: {
+      ...getRuntimeMappingsByGroupField(currentSelectedGroup),
+      'result.evaluation': {
+        type: 'keyword',
+      },
+    },
     rootAggregations: [
       {
         failedFindings: {

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx

@@ -23,6 +23,7 @@ import {
 } from '@kbn/cloud-security-posture-common';
 import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
 import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
+import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
 import { VULNERABILITY_FIELDS } from '../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script';
@@ -52,13 +53,33 @@ const getMultiFieldsSort = (sort: string[][]) => {
   });
 };
 
+const getRuntimeMappingsFromSort = (sort: string[][]) => {
+  return sort.reduce((acc, [field]) => {
+    // TODO: Add proper type for all fields available in the field selector
+    const type: RuntimePrimitiveTypes =
+      field === VULNERABILITY_FIELDS.SCORE_BASE
+        ? 'double'
+        : field === '@timestamp'
+        ? 'date'
+        : 'keyword';
+
+    return {
+      ...acc,
+      [field]: {
+        type,
+      },
+    };
+  }, {});
+};
+
 export const getVulnerabilitiesQuery = (
   { query, sort }: VulnerabilitiesQuery,
   pageParam: number
 ) => ({
   index: CDR_VULNERABILITIES_INDEX_PATTERN,
   ignore_unavailable: true,
   sort: getMultiFieldsSort(sort),
+  runtime_mappings: getRuntimeMappingsFromSort(sort),
   size: MAX_FINDINGS_TO_LOAD,
   query: {
     ...query,

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx

@@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
+/**
+ * Get runtime mappings for the given group field
+ * Some fields require additional runtime mappings to aggregate additional information
+ * Fallback to keyword type to support custom fields grouping
+ */
+const getRuntimeMappingsByGroupField = (
+  field: string
+): Record<string, { type: 'keyword' }> | undefined => {
+  switch (field) {
+    case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: {
+          type: 'keyword',
+        },
+      };
+    case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.RESOURCE_ID]: {
+          type: 'keyword',
+        },
+      };
+    case VULNERABILITY_GROUPING_OPTIONS.CVE:
+      return {
+        [VULNERABILITY_GROUPING_OPTIONS.CVE]: {
+          type: 'keyword',
+        },
+        [VULNERABILITY_FIELDS.DESCRIPTION]: {
+          type: 'keyword',
+        },
+      };
+    default:
+      return {
+        [field]: {
+          type: 'keyword',
+        },
+      };
+  }
+};
+
 /**
  * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation
  */
@@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
+    runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup),
   });
 
   const { data, isFetching } = useGroupedVulnerabilities({

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts

@@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) =
       type: 'string',
       order: direction,
       script: {
-        source: `doc["${field}"].value.toLowerCase()`,
+        source: `
+          if (doc.containsKey('${field}') && !doc['${field}'].empty) {
+            return doc['${field}'].value.toLowerCase();
+          } else {
+            return "";
+          }
+        `,
         lang: 'painless',
       },
     },