From e2bed0f502da71ed3992024d01a0407eead44704 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 11 Oct 2024 05:21:12 +1100 Subject: [PATCH] [8.x] [Cloud Security] [CDR] Handle grouping fields with missing mapping (#195702) (#195792) # Backport This will backport the following commits from `main` to `8.x`: - [[Cloud Security] [CDR] Handle grouping fields with missing mapping (#195702)](https://github.com/elastic/kibana/pull/195702) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Paulo Silva --- .../latest_findings/use_latest_findings.ts | 16 +++++ .../use_latest_findings_grouping.tsx | 72 +++++++++++++++++++ .../hooks/use_latest_vulnerabilities.tsx | 21 ++++++ .../use_latest_vulnerabilities_grouping.tsx | 46 ++++++++++++ .../utils/custom_sort_script.ts | 8 ++- 5 files changed, 162 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts index f6f27e15ee7a4..955f5a45a9743 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts @@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common'; import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest'; import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils'; @@ -39,6 +40,20 @@ interface FindingsAggs { count: estypes.AggregationsMultiBucketAggregateBase; } +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getFindingsQuery = ( { query, sort }: UseFindingsOptions, rulesStates: CspBenchmarkRulesStates, @@ -49,6 +64,7 @@ export const getFindingsQuery = ( return { index: CDR_MISCONFIGURATIONS_INDEX_PATTERN, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, aggs: getFindingsCountAggQuery(), ignore_unavailable: true, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index cc409fb95024d..6482d864347a1 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, + 'resource.id': { + type: 'keyword', + }, + 'resource.sub_type': { + type: 'keyword', + }, + 'resource.type': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.RULE_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.version': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a FindingsRootGroupingAggregation */ @@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: { + ...getRuntimeMappingsByGroupField(currentSelectedGroup), + 'result.evaluation': { + type: 'keyword', + }, + }, rootAggregations: [ { failedFindings: { diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx index 0d0ea9ba5a22f..0b9cf6978c258 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx @@ -23,6 +23,7 @@ import { } from '@kbn/cloud-security-posture-common'; import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture'; import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { VULNERABILITY_FIELDS } from '../../../common/constants'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script'; @@ -52,6 +53,25 @@ const getMultiFieldsSort = (sort: string[][]) => { }); }; +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = + field === VULNERABILITY_FIELDS.SCORE_BASE + ? 'double' + : field === '@timestamp' + ? 'date' + : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getVulnerabilitiesQuery = ( { query, sort }: VulnerabilitiesQuery, pageParam: number @@ -59,6 +79,7 @@ export const getVulnerabilitiesQuery = ( index: CDR_VULNERABILITIES_INDEX_PATTERN, ignore_unavailable: true, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, query: { ...query, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index 516cbed0c3975..3c52590f8fd80 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.RESOURCE_ID]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.CVE: + return { + [VULNERABILITY_GROUPING_OPTIONS.CVE]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.DESCRIPTION]: { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation */ @@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup), }); const { data, isFetching } = useGroupedVulnerabilities({ diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts index 780cd539305b3..e517d622e71c5 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts @@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) = type: 'string', order: direction, script: { - source: `doc["${field}"].value.toLowerCase()`, + source: ` + if (doc.containsKey('${field}') && !doc['${field}'].empty) { + return doc['${field}'].value.toLowerCase(); + } else { + return ""; + } + `, lang: 'painless', }, },