From 6efab0f01849423e52fc9f9b52f98ef8b26795f5 Mon Sep 17 00:00:00 2001 From: Paulo Silva Date: Fri, 1 Nov 2024 02:59:09 -0700 Subject: [PATCH] [Cloud Security] Update Findings page runtime fields required for third party data compatibility (#198635) ## Summary This PR enhances the solution proposed by [this PR](https://github.com/elastic/kibana/pull/195702), by picking only the fields that are currently not mapped by the current Third Party integrations, this fixes performance degradation identified during the QA cycle of 8.16.0. Fixes: - https://github.com/elastic/security-team/issues/11034 ### Misconfiguration Runtime fields - **rule.benchmark.rule_number**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when sorting by **Rule Number** column on the Misconfigurations Data Table. - **rule.section**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when sorting by **Framework Section** column on the Misconfigurations Data Table. - **resource.sub_type**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when sorting by **Resource Type** column on the Misconfigurations Data Table. - **orchestrator.cluster.name**: runtime mapping kept because this field is missing on `security_solution-wiz.misconfiguration_latest-v1` causing filtering out data when grouping by **Kubernetes Cluster** column on the Misconfigurations page. - **cloud.account.name**: runtime mapping kept because this field is missing on `security_solution-aws.misconfiguration_latest-v1` causing filtering out data when grouping by **Kubernetes Cluster** column on the Misconfigurations page. ### Vulnerability Runtime Fields: - **observer.vendor**: runtime mapping added because this field is mapped as `text` on `security_solution-wiz.vulnerability_latest-v1` causing filtering out when sorting by the **Vendor** column on the Vulnerability Data Table - **cloud.provider**: runtime mapping added because this field is mapped as `text` on `security_solution-wiz.vulnerability_latest-v1` causing filtering out when grouping by **Cloud Account** on the Vulnerability page. (This field is needed in order to retrieve the Cloud Provider name and icon) ## Screenshot - Left: After the changes / Right: Current https://github.com/user-attachments/assets/2cbdd8b7-131c-42e4-a881-632f8cd3854b https://github.com/user-attachments/assets/4372feb6-4c01-4047-a90a-d6728f9400fe https://github.com/user-attachments/assets/b9e32514-f2ee-4e4d-ba5f-ea3e20d4d0b2 --- .../common/utils/helpers.ts | 2 +- .../public/common/constants.ts | 30 ++++++++ .../public/components/cloud_provider_icon.tsx | 2 +- .../latest_findings/use_grouped_findings.tsx | 3 - .../latest_findings/use_latest_findings.ts | 22 +++--- .../use_latest_findings_grouping.tsx | 71 +++---------------- .../hooks/use_latest_vulnerabilities.tsx | 31 ++++---- .../use_latest_vulnerabilities_grouping.tsx | 42 +++-------- 8 files changed, 80 insertions(+), 123 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/common/utils/helpers.ts b/x-pack/plugins/cloud_security_posture/common/utils/helpers.ts index 90e11734d72c6..86eb6ba963402 100644 --- a/x-pack/plugins/cloud_security_posture/common/utils/helpers.ts +++ b/x-pack/plugins/cloud_security_posture/common/utils/helpers.ts @@ -214,7 +214,7 @@ export const getBenchmarkApplicableTo = (benchmarkId: BenchmarksCisId) => { }; export const getCloudProviderNameFromAbbreviation = (cloudProvider: string) => { - switch (cloudProvider) { + switch (cloudProvider.toLowerCase()) { case 'azure': return CLOUD_PROVIDER_NAMES.AZURE; case 'aws': diff --git a/x-pack/plugins/cloud_security_posture/public/common/constants.ts b/x-pack/plugins/cloud_security_posture/public/common/constants.ts index 50d191cf07167..fab73eb153e69 100644 --- a/x-pack/plugins/cloud_security_posture/public/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/public/common/constants.ts @@ -256,3 +256,33 @@ export const VULNERABILITY_GROUPING_OPTIONS = { CLOUD_ACCOUNT_NAME: VULNERABILITY_FIELDS.CLOUD_ACCOUNT_NAME, CVE: VULNERABILITY_FIELDS.VULNERABILITY_ID, }; + +/* +The fields below are default columns of the Cloud Security Data Table that need to have keyword mapping. +The runtime mappings are used to prevent filtering out the data when any of these columns are sorted in the Data Table. +TODO: Remove the fields below once they are mapped as Keyword in the Third Party integrations, or remove +the fields from the runtime mappings if they are removed from the Data Table. +*/ +export const CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS: string[] = [ + VULNERABILITY_FIELDS.VENDOR, +]; +export const CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS: string[] = [ + 'rule.benchmark.rule_number', + 'rule.section', + 'resource.sub_type', +]; + +/* +The fields below are used to group the data in the Cloud Security Data Table. +The keys are the fields that are used to group the data, and the values are the fields that need to have keyword mapping +to prevent filtering out the data when grouping by the key field. +TODO: Remove the fields below once they are mapped as Keyword in the Third Party integrations, or remove +the fields from the runtime mappings if they are removed from the Data Table. +*/ +export const CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS: Record = { + [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: [VULNERABILITY_FIELDS.CLOUD_PROVIDER], +}; +export const CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS: Record = { + [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: ['orchestrator.cluster.name'], + [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: ['cloud.account.name'], +}; diff --git a/x-pack/plugins/cloud_security_posture/public/components/cloud_provider_icon.tsx b/x-pack/plugins/cloud_security_posture/public/components/cloud_provider_icon.tsx index b6acdac0ee1b1..a022e38960894 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/cloud_provider_icon.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/cloud_provider_icon.tsx @@ -18,7 +18,7 @@ interface Props { } const getCloudProviderIcon = (cloudProvider: string) => { - switch (cloudProvider) { + switch (cloudProvider.toLowerCase()) { case 'azure': return 'logoAzure'; case 'aws': diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_grouped_findings.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_grouped_findings.tsx index 6d901a76a29c3..b98ff432a3c96 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_grouped_findings.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_grouped_findings.tsx @@ -52,9 +52,6 @@ export interface FindingsGroupingAggregation { resourceSubType?: { buckets?: GenericBuckets[]; }; - resourceType?: { - buckets?: GenericBuckets[]; - }; benchmarkName?: { buckets?: GenericBuckets[]; }; diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts index 068eb3df1b10f..64353230211bc 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts @@ -22,6 +22,7 @@ import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; +import { CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS } from '../../../common/constants'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils'; @@ -41,17 +42,18 @@ interface FindingsAggs { } const getRuntimeMappingsFromSort = (sort: string[][]) => { - return sort.reduce((acc, [field]) => { - // TODO: Add proper type for all fields available in the field selector - const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword'; + return sort + .filter(([field]) => CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS.includes(field)) + .reduce((acc, [field]) => { + const type: RuntimePrimitiveTypes = 'keyword'; - return { - ...acc, - [field]: { - type, - }, - }; - }, {}); + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); }; export const getFindingsQuery = ( diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index e009ee966fb96..45c5418ed5129 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -21,6 +21,7 @@ import { } from '@kbn/cloud-security-posture-common'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; import { + CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS, FINDINGS_GROUPING_OPTIONS, LOCAL_STORAGE_FINDINGS_GROUPING_KEY, } from '../../../common/constants'; @@ -90,7 +91,6 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { ...aggMetrics, getTermAggregation('resourceName', 'resource.id'), getTermAggregation('resourceSubType', 'resource.sub_type'), - getTermAggregation('resourceType', 'resource.type'), ]; case FINDINGS_GROUPING_OPTIONS.RULE_NAME: return [ @@ -122,62 +122,18 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { const getRuntimeMappingsByGroupField = ( field: string ): Record | undefined => { - switch (field) { - case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME: - return { - [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: { - type: 'keyword', - }, - 'resource.id': { - type: 'keyword', - }, - 'resource.sub_type': { - type: 'keyword', - }, - 'resource.type': { - type: 'keyword', - }, - }; - case FINDINGS_GROUPING_OPTIONS.RULE_NAME: - return { - [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: { - type: 'keyword', - }, - 'rule.benchmark.version': { - type: 'keyword', - }, - }; - case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: - return { - [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + if (CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS?.[field]) { + return CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS[field].reduce( + (acc, runtimeField) => ({ + ...acc, + [runtimeField]: { type: 'keyword', }, - 'rule.benchmark.name': { - type: 'keyword', - }, - 'rule.benchmark.id': { - type: 'keyword', - }, - }; - case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME: - return { - [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: { - type: 'keyword', - }, - 'rule.benchmark.name': { - type: 'keyword', - }, - 'rule.benchmark.id': { - type: 'keyword', - }, - }; - default: - return { - [field]: { - type: 'keyword', - }, - }; + }), + {} + ); } + return {}; }; /** @@ -255,12 +211,7 @@ export const useLatestFindingsGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), - runtimeMappings: { - ...getRuntimeMappingsByGroupField(currentSelectedGroup), - 'result.evaluation': { - type: 'keyword', - }, - }, + runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup), rootAggregations: [ { failedFindings: { diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx index 5f01a4693c8f5..a998707c4704f 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx @@ -24,7 +24,10 @@ import { import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture'; import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest'; import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; -import { VULNERABILITY_FIELDS } from '../../../common/constants'; +import { + CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS, + VULNERABILITY_FIELDS, +} from '../../../common/constants'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script'; type LatestFindingsRequest = IKibanaSearchRequest; @@ -54,22 +57,18 @@ const getMultiFieldsSort = (sort: string[][]) => { }; const getRuntimeMappingsFromSort = (sort: string[][]) => { - return sort.reduce((acc, [field]) => { - // TODO: Add proper type for all fields available in the field selector - const type: RuntimePrimitiveTypes = - field === VULNERABILITY_FIELDS.SCORE_BASE - ? 'double' - : field === '@timestamp' - ? 'date' - : 'keyword'; + return sort + .filter(([field]) => CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS.includes(field)) + .reduce((acc, [field]) => { + const type: RuntimePrimitiveTypes = 'keyword'; - return { - ...acc, - [field]: { - type, - }, - }; - }, {}); + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); }; export const getVulnerabilitiesQuery = ( diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index d79b4620ec899..1d73b21f083a5 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -23,6 +23,7 @@ import { LOCAL_STORAGE_VULNERABILITIES_GROUPING_KEY, VULNERABILITY_GROUPING_OPTIONS, VULNERABILITY_FIELDS, + CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS, } from '../../../common/constants'; import { useDataViewContext } from '../../../common/contexts/data_view_context'; import { @@ -102,41 +103,18 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { const getRuntimeMappingsByGroupField = ( field: string ): Record | undefined => { - switch (field) { - case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: - return { - [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { - type: 'keyword', - }, - [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: { - type: 'keyword', - }, - }; - case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME: - return { - [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: { - type: 'keyword', - }, - [VULNERABILITY_FIELDS.RESOURCE_ID]: { - type: 'keyword', - }, - }; - case VULNERABILITY_GROUPING_OPTIONS.CVE: - return { - [VULNERABILITY_GROUPING_OPTIONS.CVE]: { - type: 'keyword', - }, - [VULNERABILITY_FIELDS.DESCRIPTION]: { - type: 'keyword', - }, - }; - default: - return { - [field]: { + if (CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS?.[field]) { + return CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS[field].reduce( + (acc, runtimeField) => ({ + ...acc, + [runtimeField]: { type: 'keyword', }, - }; + }), + {} + ); } + return {}; }; /**