From de3cec42aef4070f45e13688e26d525230691ab7 Mon Sep 17 00:00:00 2001 From: Paulo Silva Date: Thu, 10 Oct 2024 07:52:49 -0700 Subject: [PATCH] [Cloud Security] [CDR] Handle grouping fields with missing mapping (#195702) ## Summary This PR fixes https://github.com/elastic/security-team/issues/10632 by adding runtime mapping support for fields that are missing in mapping, this is useful when querying a DataView that points to multiple indices where the mapping is not guaranteed to exist as it's the case with CDR that adds supports to Third Party data. Also added runtime mapping to sorted fields, as it's not guaranteed that all fields shown on the table have mapped fields. (cherry picked from commit e53e54550f9ab9ce2db83ec56a5c704a96f37355) --- .../latest_findings/use_latest_findings.ts | 16 +++++ .../use_latest_findings_grouping.tsx | 72 +++++++++++++++++++ .../hooks/use_latest_vulnerabilities.tsx | 21 ++++++ .../use_latest_vulnerabilities_grouping.tsx | 46 ++++++++++++ .../utils/custom_sort_script.ts | 8 ++- 5 files changed, 162 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts index f6f27e15ee7a4..955f5a45a9743 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts @@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common'; import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest'; import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils'; @@ -39,6 +40,20 @@ interface FindingsAggs { count: estypes.AggregationsMultiBucketAggregateBase; } +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getFindingsQuery = ( { query, sort }: UseFindingsOptions, rulesStates: CspBenchmarkRulesStates, @@ -49,6 +64,7 @@ export const getFindingsQuery = ( return { index: CDR_MISCONFIGURATIONS_INDEX_PATTERN, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, aggs: getFindingsCountAggQuery(), ignore_unavailable: true, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index cc409fb95024d..6482d864347a1 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, + 'resource.id': { + type: 'keyword', + }, + 'resource.sub_type': { + type: 'keyword', + }, + 'resource.type': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.RULE_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.version': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a FindingsRootGroupingAggregation */ @@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: { + ...getRuntimeMappingsByGroupField(currentSelectedGroup), + 'result.evaluation': { + type: 'keyword', + }, + }, rootAggregations: [ { failedFindings: { diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx index 0d0ea9ba5a22f..0b9cf6978c258 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx @@ -23,6 +23,7 @@ import { } from '@kbn/cloud-security-posture-common'; import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture'; import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { VULNERABILITY_FIELDS } from '../../../common/constants'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script'; @@ -52,6 +53,25 @@ const getMultiFieldsSort = (sort: string[][]) => { }); }; +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = + field === VULNERABILITY_FIELDS.SCORE_BASE + ? 'double' + : field === '@timestamp' + ? 'date' + : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getVulnerabilitiesQuery = ( { query, sort }: VulnerabilitiesQuery, pageParam: number @@ -59,6 +79,7 @@ export const getVulnerabilitiesQuery = ( index: CDR_VULNERABILITIES_INDEX_PATTERN, ignore_unavailable: true, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, query: { ...query, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index 516cbed0c3975..3c52590f8fd80 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.RESOURCE_ID]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.CVE: + return { + [VULNERABILITY_GROUPING_OPTIONS.CVE]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.DESCRIPTION]: { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation */ @@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup), }); const { data, isFetching } = useGroupedVulnerabilities({ diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts index 780cd539305b3..e517d622e71c5 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts @@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) = type: 'string', order: direction, script: { - source: `doc["${field}"].value.toLowerCase()`, + source: ` + if (doc.containsKey('${field}') && !doc['${field}'].empty) { + return doc['${field}'].value.toLowerCase(); + } else { + return ""; + } + `, lang: 'painless', }, },