[Cloud Security] Update Findings page runtime fields required for thi…

…rd party data compatibility (elastic#198635) ## Summary This PR enhances the solution proposed by [this PR](elastic#195702), by picking only the fields that are currently not mapped by the current Third Party integrations, this fixes performance degradation identified during the QA cycle of 8.16.0. Fixes: - elastic/security-team#11034 ### Misconfiguration Runtime fields - **rule.benchmark.rule_number**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when sorting by **Rule Number** column on the Misconfigurations Data Table. - **rule.section**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when sorting by **Framework Section** column on the Misconfigurations Data Table. - **resource.sub_type**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when sorting by **Resource Type** column on the Misconfigurations Data Table. - **orchestrator.cluster.name**: runtime mapping kept because this field is missing on `security_solution-wiz.misconfiguration_latest-v1` causing filtering out data when grouping by **Kubernetes Cluster** column on the Misconfigurations page. - **cloud.account.name**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when grouping by **Kubernetes Cluster** column on the Misconfigurations page. ### Vulnerability Runtime Fields: - **observer.vendor**: runtime mapping added because this field is mapped as `text` on `security_solution-wiz.vulnerability_latest-v1` causing filtering out when sorting by the **Vendor** column on the Vulnerability Data Table - **cloud.provider**: runtime mapping added because this field is mapped as `text` on `security_solution-wiz.vulnerability_latest-v1` causing filtering out when grouping by **Cloud Account** on the Vulnerability page. (This field is needed in order to retrieve the Cloud Provider name and icon) ## Screenshot - Left: After the changes / Right: Current https://github.com/user-attachments/assets/2cbdd8b7-131c-42e4-a881-632f8cd3854b https://github.com/user-attachments/assets/4372feb6-4c01-4047-a90a-d6728f9400fe https://github.com/user-attachments/assets/b9e32514-f2ee-4e4d-ba5f-ea3e20d4d0b2
nreese · Nov 1, 2024 · 6efab0f · 6efab0f
1 parent 19fb1e3
commit 6efab0f
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 123 deletions.
diff --git a/x-pack/plugins/cloud_security_posture/common/utils/helpers.ts b/x-pack/plugins/cloud_security_posture/common/utils/helpers.ts
@@ -214,7 +214,7 @@ export const getBenchmarkApplicableTo = (benchmarkId: BenchmarksCisId) => {
 };
 
 export const getCloudProviderNameFromAbbreviation = (cloudProvider: string) => {
-  switch (cloudProvider) {
+  switch (cloudProvider.toLowerCase()) {
     case 'azure':
       return CLOUD_PROVIDER_NAMES.AZURE;
     case 'aws':

diff --git a/x-pack/plugins/cloud_security_posture/public/common/constants.ts b/x-pack/plugins/cloud_security_posture/public/common/constants.ts
@@ -256,3 +256,33 @@ export const VULNERABILITY_GROUPING_OPTIONS = {
   CLOUD_ACCOUNT_NAME: VULNERABILITY_FIELDS.CLOUD_ACCOUNT_NAME,
   CVE: VULNERABILITY_FIELDS.VULNERABILITY_ID,
 };
+
+/*
+The fields below are default columns of the Cloud Security Data Table that need to have keyword mapping.
+The runtime mappings are used to prevent filtering out the data when any of these columns are sorted in the Data Table.
+TODO: Remove the fields below once they are mapped as Keyword in the Third Party integrations, or remove
+the fields from the runtime mappings if they are removed from the Data Table.
+*/
+export const CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS: string[] = [
+  VULNERABILITY_FIELDS.VENDOR,
+];
+export const CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS: string[] = [
+  'rule.benchmark.rule_number',
+  'rule.section',
+  'resource.sub_type',
+];
+
+/*
+The fields below are used to group the data in the Cloud Security Data Table.
+The keys are the fields that are used to group the data, and the values are the fields that need to have keyword mapping
+to prevent filtering out the data when grouping by the key field.
+TODO: Remove the fields below once they are mapped as Keyword in the Third Party integrations, or remove
+the fields from the runtime mappings if they are removed from the Data Table.
+*/
+export const CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS: Record<string, string[]> = {
+  [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: [VULNERABILITY_FIELDS.CLOUD_PROVIDER],
+};
+export const CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS: Record<string, string[]> = {
+  [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: ['orchestrator.cluster.name'],
+  [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: ['cloud.account.name'],
+};
diff --git a/x-pack/plugins/cloud_security_posture/public/components/cloud_provider_icon.tsx b/x-pack/plugins/cloud_security_posture/public/components/cloud_provider_icon.tsx
@@ -18,7 +18,7 @@ interface Props {
 }
 
 const getCloudProviderIcon = (cloudProvider: string) => {
-  switch (cloudProvider) {
+  switch (cloudProvider.toLowerCase()) {
     case 'azure':
       return 'logoAzure';
     case 'aws':

diff --git a/...oud_security_posture/public/pages/configurations/latest_findings/use_grouped_findings.tsx b/...oud_security_posture/public/pages/configurations/latest_findings/use_grouped_findings.tsx
@@ -52,9 +52,6 @@ export interface FindingsGroupingAggregation {
   resourceSubType?: {
     buckets?: GenericBuckets[];
   };
-  resourceType?: {
-    buckets?: GenericBuckets[];
-  };
   benchmarkName?: {
     buckets?: GenericBuckets[];
   };

diff --git a/...cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/...cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts
@@ -22,6 +22,7 @@ import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common
 import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
 import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
+import { CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS } from '../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils';
 
@@ -41,17 +42,18 @@ interface FindingsAggs {
 }
 
 const getRuntimeMappingsFromSort = (sort: string[][]) => {
-  return sort.reduce((acc, [field]) => {
-    // TODO: Add proper type for all fields available in the field selector
-    const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword';
+  return sort
+    .filter(([field]) => CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS.includes(field))
+    .reduce((acc, [field]) => {
+      const type: RuntimePrimitiveTypes = 'keyword';
 
-    return {
-      ...acc,
-      [field]: {
-        type,
-      },
-    };
-  }, {});
+      return {
+        ...acc,
+        [field]: {
+          type,
+        },
+      };
+    }, {});
 };
 
 export const getFindingsQuery = (

diff --git a/...rity_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/...rity_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx
@@ -21,6 +21,7 @@ import {
 } from '@kbn/cloud-security-posture-common';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
 import {
+  CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS,
   FINDINGS_GROUPING_OPTIONS,
   LOCAL_STORAGE_FINDINGS_GROUPING_KEY,
 } from '../../../common/constants';
@@ -90,7 +91,6 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
         ...aggMetrics,
         getTermAggregation('resourceName', 'resource.id'),
         getTermAggregation('resourceSubType', 'resource.sub_type'),
-        getTermAggregation('resourceType', 'resource.type'),
       ];
     case FINDINGS_GROUPING_OPTIONS.RULE_NAME:
       return [
@@ -122,62 +122,18 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
 const getRuntimeMappingsByGroupField = (
   field: string
 ): Record<string, { type: 'keyword' }> | undefined => {
-  switch (field) {
-    case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME:
-      return {
-        [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: {
-          type: 'keyword',
-        },
-        'resource.id': {
-          type: 'keyword',
-        },
-        'resource.sub_type': {
-          type: 'keyword',
-        },
-        'resource.type': {
-          type: 'keyword',
-        },
-      };
-    case FINDINGS_GROUPING_OPTIONS.RULE_NAME:
-      return {
-        [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: {
-          type: 'keyword',
-        },
-        'rule.benchmark.version': {
-          type: 'keyword',
-        },
-      };
-    case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
-      return {
-        [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
+  if (CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS?.[field]) {
+    return CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS[field].reduce(
+      (acc, runtimeField) => ({
+        ...acc,
+        [runtimeField]: {
           type: 'keyword',
         },
-        'rule.benchmark.name': {
-          type: 'keyword',
-        },
-        'rule.benchmark.id': {
-          type: 'keyword',
-        },
-      };
-    case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME:
-      return {
-        [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: {
-          type: 'keyword',
-        },
-        'rule.benchmark.name': {
-          type: 'keyword',
-        },
-        'rule.benchmark.id': {
-          type: 'keyword',
-        },
-      };
-    default:
-      return {
-        [field]: {
-          type: 'keyword',
-        },
-      };
+      }),
+      {}
+    );
   }
+  return {};
 };
 
 /**
@@ -255,12 +211,7 @@ export const useLatestFindingsGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
-    runtimeMappings: {
-      ...getRuntimeMappingsByGroupField(currentSelectedGroup),
-      'result.evaluation': {
-        type: 'keyword',
-      },
-    },
+    runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup),
     rootAggregations: [
       {
         failedFindings: {

diff --git a/.../cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/.../cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx
@@ -24,7 +24,10 @@ import {
 import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
 import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
 import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
-import { VULNERABILITY_FIELDS } from '../../../common/constants';
+import {
+  CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS,
+  VULNERABILITY_FIELDS,
+} from '../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script';
 type LatestFindingsRequest = IKibanaSearchRequest<SearchRequest>;
@@ -54,22 +57,18 @@ const getMultiFieldsSort = (sort: string[][]) => {
 };
 
 const getRuntimeMappingsFromSort = (sort: string[][]) => {
-  return sort.reduce((acc, [field]) => {
-    // TODO: Add proper type for all fields available in the field selector
-    const type: RuntimePrimitiveTypes =
-      field === VULNERABILITY_FIELDS.SCORE_BASE
-        ? 'double'
-        : field === '@timestamp'
-        ? 'date'
-        : 'keyword';
+  return sort
+    .filter(([field]) => CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS.includes(field))
+    .reduce((acc, [field]) => {
+      const type: RuntimePrimitiveTypes = 'keyword';
 
-    return {
-      ...acc,
-      [field]: {
-        type,
-      },
-    };
-  }, {});
+      return {
+        ...acc,
+        [field]: {
+          type,
+        },
+      };
+    }, {});
 };
 
 export const getVulnerabilitiesQuery = (

diff --git a/...curity_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/...curity_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx
@@ -23,6 +23,7 @@ import {
   LOCAL_STORAGE_VULNERABILITIES_GROUPING_KEY,
   VULNERABILITY_GROUPING_OPTIONS,
   VULNERABILITY_FIELDS,
+  CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS,
 } from '../../../common/constants';
 import { useDataViewContext } from '../../../common/contexts/data_view_context';
 import {
@@ -102,41 +103,18 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
 const getRuntimeMappingsByGroupField = (
   field: string
 ): Record<string, { type: 'keyword' }> | undefined => {
-  switch (field) {
-    case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
-      return {
-        [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
-          type: 'keyword',
-        },
-        [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: {
-          type: 'keyword',
-        },
-      };
-    case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME:
-      return {
-        [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: {
-          type: 'keyword',
-        },
-        [VULNERABILITY_FIELDS.RESOURCE_ID]: {
-          type: 'keyword',
-        },
-      };
-    case VULNERABILITY_GROUPING_OPTIONS.CVE:
-      return {
-        [VULNERABILITY_GROUPING_OPTIONS.CVE]: {
-          type: 'keyword',
-        },
-        [VULNERABILITY_FIELDS.DESCRIPTION]: {
-          type: 'keyword',
-        },
-      };
-    default:
-      return {
-        [field]: {
+  if (CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS?.[field]) {
+    return CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS[field].reduce(
+      (acc, runtimeField) => ({
+        ...acc,
+        [runtimeField]: {
           type: 'keyword',
         },
-      };
+      }),
+      {}
+    );
   }
+  return {};
 };
 
 /**