From ad360403bca9a6072f67032f52c80eab98f7004e Mon Sep 17 00:00:00 2001 From: Karen Grigoryan Date: Mon, 26 Aug 2024 16:12:33 +0200 Subject: [PATCH] [Security Solution][DQD][Tech Debt] Refactor top level helpers (#191233) addresses https://github.com/elastic/kibana/issues/190964 Third in the series of PRs to address general DQD tech debt This one builds on previous 2 PRs https://github.com/elastic/kibana/pull/190970 https://github.com/elastic/kibana/pull/190978 Gist of changes: - split top level helpers into series of utils/* files - each utils/ file is named after common behavior it export or works with. - cleanup dead code --- .../impl/data_quality_panel/constants.ts | 4 + .../indices_details/index.test.tsx | 2 +- .../indices_details/pattern/helpers.test.ts | 4 +- .../indices_details/pattern/helpers.ts | 2 +- .../pattern/hooks/use_ilm_explain/index.tsx | 2 +- .../pattern/hooks/use_stats/index.tsx | 2 +- .../indices_details/pattern/index.test.tsx | 2 +- .../indices_details/pattern/index.tsx | 13 +- .../pattern/index_check_flyout/index.tsx | 3 +- .../index_properties/helpers.test.ts | 44 +- .../index_properties/helpers.ts | 12 +- .../index_properties/index.test.tsx | 2 +- .../get_common_table_columns/index.tsx | 2 +- .../tabs/custom_tab/helpers.test.ts | 2 +- .../index_check_fields/tabs/helpers.tsx | 3 +- .../tabs/incompatible_tab/helpers.test.ts | 2 +- .../tabs/incompatible_tab/helpers.ts | 2 +- .../tabs/same_family_tab/helpers.test.ts | 2 +- .../index_stats_panel/index.tsx | 2 +- .../index_properties/markdown/helpers.test.ts | 2 +- .../index_properties/markdown/helpers.ts | 10 +- .../utils/get_is_in_same_family.test.ts | 40 + .../utils/get_is_in_same_family.ts | 43 + .../index_properties/utils/metadata.test.ts | 402 +++++ .../index_properties/utils/metadata.ts | 167 ++ .../pattern_label/ilm_phase_counts/index.tsx | 2 +- .../ilm_phase_counts/translations.ts | 53 + .../get_pattern_ilm_phase_description.test.ts | 106 ++ .../get_pattern_ilm_phase_description.ts | 33 + .../pattern/summary_table/helpers.test.tsx | 2 +- .../pattern/summary_table/helpers.tsx | 3 +- .../pattern/summary_table/index.test.tsx | 2 +- .../pattern/utils/stats.test.ts | 234 +++ .../indices_details/pattern/utils/stats.ts | 72 + .../storage_details/helpers.test.ts | 2 +- .../storage_details/helpers.ts | 2 +- .../storage_details/index.test.tsx | 2 +- .../storage_treemap/index.test.tsx | 2 +- .../ilm_phase_filter/index.tsx | 2 +- .../data_quality_summary/index.test.tsx | 4 +- .../summary_actions/check_all/helpers.ts | 2 +- .../summary_actions/check_all/index.test.tsx | 2 +- .../summary_actions/index.test.tsx | 4 +- .../summary_actions/index.tsx | 3 +- .../utils/get_error_summaries.test.ts | 188 ++ .../utils/get_error_summaries.ts | 56 + .../impl/data_quality_panel/helpers.test.ts | 1537 ----------------- .../impl/data_quality_panel/helpers.ts | 615 ------- .../hooks/use_results_rollup/constants.ts | 10 + .../hooks/use_results_rollup/index.tsx | 28 +- .../hooks/use_results_rollup/types.ts | 35 +- ..._rollups_with_latest_check_result.test.ts} | 153 +- ...attern_rollups_with_latest_check_result.ts | 92 + .../use_results_rollup/utils/stats.test.ts | 231 +++ .../{helpers.ts => utils/stats.ts} | 104 +- .../use_results_rollup/utils/storage.test.ts | 202 +++ .../hooks/use_results_rollup/utils/storage.ts | 157 ++ .../impl/data_quality_panel/index.tsx | 3 +- .../get_merged_data_quality_context_props.ts | 2 +- .../data_quality_panel/stats_rollup/index.tsx | 3 +- .../impl/data_quality_panel/translations.ts | 45 - .../impl/data_quality_panel/types.ts | 8 - .../utils/check_index.test.ts | 3 +- .../utils/fetch_mappings.ts | 2 +- .../utils/fetch_unallowed_values.test.ts | 2 +- .../utils/fetch_unallowed_values.ts | 6 +- .../utils/get_ilm_phase_description.test.ts | 55 + .../utils/get_ilm_phase_description.ts | 28 + .../get_incompatible_stat_badge_color.test.ts | 28 + .../get_incompatible_stat_badge_color.ts | 9 + .../data_quality_panel/utils/stats.test.ts | 252 +++ .../impl/data_quality_panel/utils/stats.ts | 47 + .../ecs_data_quality_dashboard/index.ts | 2 +- 73 files changed, 2644 insertions(+), 2562 deletions(-) create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/translations.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.ts delete mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.test.ts delete mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/constants.ts rename x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/{helpers.test.ts => utils/get_pattern_rollups_with_latest_check_result.test.ts} (77%) create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.test.ts rename x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/{helpers.ts => utils/stats.ts} (52%) create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.test.ts create mode 100644 x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.ts diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/constants.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/constants.ts index c89c4dc5daa15..26fe2698f077c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/constants.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/constants.ts @@ -38,3 +38,7 @@ export const ilmPhaseOptionsStatic: EuiComboBoxOptionOption[] = [ value: 'unmanaged', }, ]; + +export const EMPTY_STAT = '--'; + +export const INTERNAL_API_VERSION = '1'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/index.test.tsx index 65fc799bd3e41..d5aaa1eea19ae 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/index.test.tsx @@ -9,7 +9,7 @@ import numeral from '@elastic/numeral'; import { render, screen, waitFor } from '@testing-library/react'; import React from 'react'; -import { EMPTY_STAT } from '../../helpers'; +import { EMPTY_STAT } from '../../constants'; import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup'; import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { packetbeatNoResults } from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.test.ts index d2d81348c39ec..56dc6364ba845 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.test.ts @@ -29,8 +29,8 @@ import { mockDataQualityCheckResult } from '../../../mock/data_quality_check_res import { auditbeatWithAllResults } from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { mockStats } from '../../../mock/stats/mock_stats'; import { DataQualityCheckResult } from '../../../types'; -import { getIndexNames, getTotalDocsCount } from '../../../helpers'; import { IndexSummaryTableItem } from './types'; +import { getIndexNames, getPatternDocsCount } from './utils/stats'; const hot: IlmExplainLifecycleLifecycleExplainManaged = { index: '.ds-packetbeat-8.6.1-2023.02.04-000001', @@ -569,7 +569,7 @@ describe('helpers', () => { ilmPhases: ['hot', 'unmanaged'], isILMAvailable, }); - const newDocsCount = getTotalDocsCount({ indexNames: newIndexNames, stats: mockStats }); + const newDocsCount = getPatternDocsCount({ indexNames: newIndexNames, stats: mockStats }); test('it returns false when the `patternRollup.docsCount` equals newDocsCount', () => { expect( shouldCreatePatternRollup({ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.ts index b9aede7d799ef..5470854fd2ff0 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.ts @@ -16,8 +16,8 @@ import type { SortConfig, MeteringStatsIndex, } from '../../../types'; -import { getDocsCount, getSizeInBytes } from '../../../helpers'; import { IndexSummaryTableItem } from './types'; +import { getDocsCount, getSizeInBytes } from '../../../utils/stats'; export const isManaged = ( ilmExplainRecord: IlmExplainLifecycleLifecycleExplain | undefined diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_ilm_explain/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_ilm_explain/index.tsx index 4e46df6c67ccc..17d8b8d46dea7 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_ilm_explain/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_ilm_explain/index.tsx @@ -9,7 +9,7 @@ import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch import { useEffect, useState } from 'react'; import { useDataQualityContext } from '../../../../../data_quality_context'; -import { INTERNAL_API_VERSION } from '../../../../../helpers'; +import { INTERNAL_API_VERSION } from '../../../../../constants'; import * as i18n from '../../../../../translations'; import { useIsMounted } from '../../../../../hooks/use_is_mounted'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.tsx index e2505a6ed2237..2f1507ed47007 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.tsx @@ -10,7 +10,7 @@ import { HttpFetchQuery } from '@kbn/core/public'; import { useDataQualityContext } from '../../../../../data_quality_context'; import * as i18n from '../../../../../translations'; -import { INTERNAL_API_VERSION } from '../../../../../helpers'; +import { INTERNAL_API_VERSION } from '../../../../../constants'; import { MeteringStatsIndex } from '../../../../../types'; import { useIsMounted } from '../../../../../hooks/use_is_mounted'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.test.tsx index 43054d9afd813..cbed456f9cdd5 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.test.tsx @@ -9,7 +9,7 @@ import numeral from '@elastic/numeral'; import { render, screen } from '@testing-library/react'; import React, { ComponentProps } from 'react'; -import { EMPTY_STAT } from '../../../helpers'; +import { EMPTY_STAT } from '../../../constants'; import { TestDataQualityProviders, TestExternalProviders, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.tsx index 31db5e84a68f6..a59fe7f87d3a5 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.tsx @@ -18,13 +18,8 @@ import { shouldCreateIndexNames, shouldCreatePatternRollup, } from './helpers'; -import { - getIndexNames, - getTotalDocsCount, - getTotalPatternIncompatible, - getTotalPatternIndicesChecked, - getTotalSizeInBytes, -} from '../../../helpers'; +import { getTotalPatternIncompatible, getTotalPatternIndicesChecked } from '../../../utils/stats'; +import { getIndexNames, getPatternDocsCount, getPatternSizeInBytes } from './utils/stats'; import { LoadingEmptyPrompt } from './loading_empty_prompt'; import { PatternSummary } from './pattern_summary'; import { RemoteClustersCallout } from './remote_clusters_callout'; @@ -152,7 +147,7 @@ const PatternComponent: React.FC = ({ useEffect(() => { const newIndexNames = getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }); - const newDocsCount = getTotalDocsCount({ indexNames: newIndexNames, stats }); + const newDocsCount = getPatternDocsCount({ indexNames: newIndexNames, stats }); if ( shouldCreateIndexNames({ @@ -188,7 +183,7 @@ const PatternComponent: React.FC = ({ pattern, results: undefined, sizeInBytes: isILMAvailable - ? getTotalSizeInBytes({ + ? getPatternSizeInBytes({ indexNames: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }), stats, }) ?? 0 diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index.tsx index a08aeaaaff2d8..6748fd0651799 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index.tsx @@ -21,9 +21,10 @@ import { } from '@elastic/eui'; import React, { useCallback, useEffect } from 'react'; import moment from 'moment'; +import { getDocsCount, getSizeInBytes } from '../../../../utils/stats'; import { useIndicesCheckContext } from '../../../../contexts/indices_check_context'; -import { EMPTY_STAT, getDocsCount, getSizeInBytes } from '../../../../helpers'; +import { EMPTY_STAT } from '../../../../constants'; import { MeteringStatsIndex, PatternRollup } from '../../../../types'; import { useDataQualityContext } from '../../../../data_quality_context'; import { IndexProperties } from './index_properties'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.test.ts index bd193dce23441..962fa7a825714 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.test.ts @@ -5,11 +5,7 @@ * 2.0. */ -import { - getMappingsProperties, - getSortedPartitionedFieldMetadata, - hasAllDataFetchingCompleted, -} from './helpers'; +import { getMappingsProperties, getSortedPartitionedFieldMetadata } from './helpers'; import { mockIndicesGetMappingIndexMappingRecords } from '../../../../../mock/indices_get_mapping_index_mapping_record/mock_indices_get_mapping_index_mapping_record'; import { mockMappingsProperties } from '../../../../../mock/mappings_properties/mock_mappings_properties'; import { EcsFlatTyped } from '../../../../../constants'; @@ -250,42 +246,4 @@ describe('helpers', () => { ).toBeNull(); }); }); - - describe('hasAllDataFetchingCompleted', () => { - test('it returns false when both the mappings and unallowed values are loading', () => { - expect( - hasAllDataFetchingCompleted({ - loadingMappings: true, - loadingUnallowedValues: true, - }) - ).toBe(false); - }); - - test('it returns false when mappings are loading, and unallowed values are NOT loading', () => { - expect( - hasAllDataFetchingCompleted({ - loadingMappings: true, - loadingUnallowedValues: false, - }) - ).toBe(false); - }); - - test('it returns false when mappings are NOT loading, and unallowed values are loading', () => { - expect( - hasAllDataFetchingCompleted({ - loadingMappings: false, - loadingUnallowedValues: true, - }) - ).toBe(false); - }); - - test('it returns true when both the mappings and unallowed values have finished loading', () => { - expect( - hasAllDataFetchingCompleted({ - loadingMappings: false, - loadingUnallowedValues: false, - }) - ).toBe(true); - }); - }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.ts index 4097186be6b73..cf4ae6562b8ed 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.ts @@ -12,13 +12,13 @@ import type { import { sortBy } from 'lodash/fp'; import { EcsFlatTyped } from '../../../../../constants'; +import type { PartitionedFieldMetadata, UnallowedValueCount } from '../../../../../types'; import { getEnrichedFieldMetadata, getFieldTypes, getMissingTimestampFieldMetadata, getPartitionedFieldMetadata, -} from '../../../../../helpers'; -import type { PartitionedFieldMetadata, UnallowedValueCount } from '../../../../../types'; +} from './utils/metadata'; export const ALL_TAB_ID = 'allTab'; export const ECS_COMPLIANT_TAB_ID = 'ecsCompliantTab'; @@ -90,11 +90,3 @@ export const getMappingsProperties = ({ return null; }; - -export const hasAllDataFetchingCompleted = ({ - loadingMappings, - loadingUnallowedValues, -}: { - loadingMappings: boolean; - loadingUnallowedValues: boolean; -}): boolean => loadingMappings === false && loadingUnallowedValues === false; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index.test.tsx index 1427b67cd41d3..2d361ec0ed34f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index.test.tsx @@ -9,7 +9,7 @@ import numeral from '@elastic/numeral'; import { render, screen, waitFor } from '@testing-library/react'; import React from 'react'; -import { EMPTY_STAT } from '../../../../../helpers'; +import { EMPTY_STAT } from '../../../../../constants'; import { auditbeatWithAllResults } from '../../../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { TestDataQualityProviders, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/compare_fields_table/get_common_table_columns/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/compare_fields_table/get_common_table_columns/index.tsx index d4194790589db..757294cc89c6d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/compare_fields_table/get_common_table_columns/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/compare_fields_table/get_common_table_columns/index.tsx @@ -11,7 +11,6 @@ import React from 'react'; import { SameFamily } from '../same_family'; import { EcsAllowedValues } from '../ecs_allowed_values'; -import { getIsInSameFamily } from '../../../../../../../../../helpers'; import { IndexInvalidValues } from '../index_invalid_values'; import { CodeDanger, CodeSuccess } from '../../../../../../../../../styles'; import * as i18n from '../translations'; @@ -20,6 +19,7 @@ import type { EnrichedFieldMetadata, UnallowedValueCount, } from '../../../../../../../../../types'; +import { getIsInSameFamily } from '../../../../utils/get_is_in_same_family'; export const EMPTY_PLACEHOLDER = '--'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/custom_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/custom_tab/helpers.test.ts index 1813db953f8a7..5061a818e17fd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/custom_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/custom_tab/helpers.test.ts @@ -19,7 +19,7 @@ import { someField, } from '../../../../../../../../mock/enriched_field_metadata/mock_enriched_field_metadata'; import { mockPartitionedFieldMetadata } from '../../../../../../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; -import { EMPTY_STAT } from '../../../../../../../../helpers'; +import { EMPTY_STAT } from '../../../../../../../../constants'; const defaultBytesFormat = '0,0.[0]b'; const formatBytes = (value: number | undefined) => diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/helpers.tsx index 6db808d312d13..fa4e63afc6f3b 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/helpers.tsx @@ -9,10 +9,11 @@ import { EuiBadge } from '@elastic/eui'; import React from 'react'; import styled from 'styled-components'; +import { getSizeInBytes } from '../../../../../../../utils/stats'; +import { getIncompatibleStatBadgeColor } from '../../../../../../../utils/get_incompatible_stat_badge_color'; import { AllTab } from './all_tab'; import { CustomTab } from './custom_tab'; import { EcsCompliantTab } from './ecs_compliant_tab'; -import { getIncompatibleStatBadgeColor, getSizeInBytes } from '../../../../../../../helpers'; import { IncompatibleTab } from './incompatible_tab'; import { ALL_TAB_ID, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.test.ts index e4ee7eb3fdb99..5d0e66daff88a 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.test.ts @@ -18,7 +18,7 @@ import { getIncompatibleValuesFields, showInvalidCallout, } from './helpers'; -import { EMPTY_STAT } from '../../../../../../../../helpers'; +import { EMPTY_STAT } from '../../../../../../../../constants'; import { DETECTION_ENGINE_RULES_MAY_NOT_MATCH, MAPPINGS_THAT_CONFLICT_WITH_ECS, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.ts index 957cf6fcb5344..edd7152c6d492 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.ts @@ -34,7 +34,7 @@ import { DOCUMENT_VALUES_ACTUAL, ECS_VALUES_EXPECTED, } from '../compare_fields_table/translations'; -import { getIsInSameFamily } from '../../../../../../../../helpers'; +import { getIsInSameFamily } from '../../../utils/get_is_in_same_family'; export const getIncompatibleFieldsMarkdownComment = (incompatible: number): string => getMarkdownComment({ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/same_family_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/same_family_tab/helpers.test.ts index d5f7101cc8acf..6eedd81fae4a5 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/same_family_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/same_family_tab/helpers.test.ts @@ -13,7 +13,7 @@ import { getSameFamilyMarkdownComment, getSameFamilyMarkdownTablesComment, } from './helpers'; -import { EMPTY_STAT } from '../../../../../../../../helpers'; +import { EMPTY_STAT } from '../../../../../../../../constants'; import { mockPartitionedFieldMetadata } from '../../../../../../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; import { mockPartitionedFieldMetadataWithSameFamily } from '../../../../../../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_stats_panel/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_stats_panel/index.tsx index 9fb1ca590a266..da5ff8efc4242 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_stats_panel/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_stats_panel/index.tsx @@ -13,7 +13,7 @@ import { DOCS } from '../translations'; import { ILM_PHASE } from '../../../../../../translations'; import { SIZE } from '../../../summary_table/translations'; import { Stat } from '../../../../../../stat'; -import { getIlmPhaseDescription } from '../../../../../../helpers'; +import { getIlmPhaseDescription } from '../../../../../../utils/get_ilm_phase_description'; const StyledFlexItem = styled(EuiFlexItem)` border-right: 1px solid ${({ theme }) => theme.eui.euiBorderColor}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.test.ts index e63044346bd0d..5fe3dbed0bd25 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.test.ts @@ -45,7 +45,7 @@ import { getSummaryTableMarkdownRow, getTabCountsMarkdownComment, } from './helpers'; -import { EMPTY_STAT } from '../../../../../../helpers'; +import { EMPTY_STAT } from '../../../../../../constants'; import { mockAllowedValues } from '../../../../../../mock/allowed_values/mock_allowed_values'; import { eventCategory, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.ts index c8181e14dd685..cc32fab713881 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.ts @@ -5,6 +5,10 @@ * 2.0. */ +import { + getTotalPatternIncompatible, + getTotalPatternIndicesChecked, +} from '../../../../../../utils/stats'; import { ERRORS_MAY_OCCUR, ERRORS_CALLOUT_SUMMARY, @@ -15,11 +19,7 @@ import { THE_FOLLOWING_PRIVILEGES_ARE_REQUIRED, VIEW_INDEX_METADATA, } from '../../../../../../data_quality_summary/summary_actions/check_status/errors_popover/translations'; -import { - EMPTY_STAT, - getTotalPatternIncompatible, - getTotalPatternIndicesChecked, -} from '../../../../../../helpers'; +import { EMPTY_STAT } from '../../../../../../constants'; import { SAME_FAMILY } from '../index_check_fields/tabs/compare_fields_table/same_family/translations'; import { HOT, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.test.ts new file mode 100644 index 0000000000000..63388b15c9495 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getIsInSameFamily } from './get_is_in_same_family'; + +describe('getIsInSameFamily', () => { + test('it returns false when ecsExpectedType is undefined', () => { + expect(getIsInSameFamily({ ecsExpectedType: undefined, type: 'keyword' })).toBe(false); + }); + + const expectedFamilyMembers: { + [key: string]: string[]; + } = { + constant_keyword: ['keyword', 'wildcard'], // `keyword` and `wildcard` in the same family as `constant_keyword` + keyword: ['constant_keyword', 'wildcard'], + match_only_text: ['text'], + text: ['match_only_text'], + wildcard: ['keyword', 'constant_keyword'], + }; + + const ecsExpectedTypes = Object.keys(expectedFamilyMembers); + + ecsExpectedTypes.forEach((ecsExpectedType) => { + const otherMembersOfSameFamily = expectedFamilyMembers[ecsExpectedType]; + + otherMembersOfSameFamily.forEach((type) => + test(`it returns true for ecsExpectedType '${ecsExpectedType}' when given '${type}', a type in the same family`, () => { + expect(getIsInSameFamily({ ecsExpectedType, type })).toBe(true); + }) + ); + + test(`it returns false for ecsExpectedType '${ecsExpectedType}' when given 'date', a type NOT in the same family`, () => { + expect(getIsInSameFamily({ ecsExpectedType, type: 'date' })).toBe(false); + }); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.ts new file mode 100644 index 0000000000000..aa56bc472cbd0 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Per https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html#_core_datatypes + * + * ``` + * Field types are grouped by _family_. Types in the same family have exactly + * the same search behavior but may have different space usage or + * performance characteristics. + * + * Currently, there are two type families, `keyword` and `text`. Other type + * families have only a single field type. For example, the `boolean` type + * family consists of one field type: `boolean`. + * ``` + */ +export const fieldTypeFamilies: Record> = { + keyword: new Set(['keyword', 'constant_keyword', 'wildcard']), + text: new Set(['text', 'match_only_text']), +}; + +export const getIsInSameFamily = ({ + ecsExpectedType, + type, +}: { + ecsExpectedType: string | undefined; + type: string; +}): boolean => { + if (ecsExpectedType != null) { + const allFamilies = Object.values(fieldTypeFamilies); + + return allFamilies.reduce( + (acc, family) => (acc !== true ? family.has(ecsExpectedType) && family.has(type) : acc), + false + ); + } else { + return false; + } +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.test.ts new file mode 100644 index 0000000000000..d59e4821ed1c8 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.test.ts @@ -0,0 +1,402 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { omit } from 'lodash/fp'; + +import { + EnrichedFieldMetadata, + PartitionedFieldMetadata, + UnallowedValueCount, +} from '../../../../../../types'; +import { mockMappingsProperties } from '../../../../../../mock/mappings_properties/mock_mappings_properties'; +import { + FieldType, + getEnrichedFieldMetadata, + getFieldTypes, + getMissingTimestampFieldMetadata, + getPartitionedFieldMetadata, + isMappingCompatible, +} from './metadata'; +import { EcsFlatTyped } from '../../../../../../constants'; +import { + hostNameWithTextMapping, + hostNameKeyword, + someField, + someFieldKeyword, + sourceIpWithTextMapping, + sourceIpKeyword, + sourcePort, + timestamp, + eventCategoryWithUnallowedValues, +} from '../../../../../../mock/enriched_field_metadata/mock_enriched_field_metadata'; + +describe('getFieldTypes', () => { + const expected = [ + { + field: '@timestamp', + type: 'date', + }, + { + field: 'event.category', + type: 'keyword', + }, + { + field: 'host.name', + type: 'text', + }, + { + field: 'host.name.keyword', + type: 'keyword', + }, + { + field: 'some.field', + type: 'text', + }, + { + field: 'some.field.keyword', + type: 'keyword', + }, + { + field: 'source.ip', + type: 'text', + }, + { + field: 'source.ip.keyword', + type: 'keyword', + }, + { + field: 'source.port', + type: 'long', + }, + ]; + + test('it flattens the field names and types in the mapping properties', () => { + expect(getFieldTypes(mockMappingsProperties)).toEqual(expected); + }); + + test('it throws a type error when mappingsProperties is not flatten-able', () => { + // @ts-expect-error + const invalidType: Record = []; // <-- this is an array, NOT a valid Record + + expect(() => getFieldTypes(invalidType)).toThrowError('Root value is not flatten-able'); + }); +}); + +describe('isMappingCompatible', () => { + test('it returns true for an exact match', () => { + expect(isMappingCompatible({ ecsExpectedType: 'keyword', type: 'keyword' })).toBe(true); + }); + + test("it returns false when both types don't exactly match", () => { + expect(isMappingCompatible({ ecsExpectedType: 'wildcard', type: 'keyword' })).toBe(false); + }); +}); + +describe('getEnrichedFieldMetadata', () => { + /** + * The ECS schema + * https://raw.githubusercontent.com/elastic/ecs/main/generated/ecs/ecs_flat.yml + * defines a handful of fields that have `allowed_values`. For these + * fields, the documents in an index should only have specific values. + * + * This instance of the type `Record` + * represents an index that doesn't have any unallowed values, for the + * specified keys in the map, i.e. `event.category`, `event.kind`, etc. + * + * This will be used to test the happy path. Variants of this + * value will be used to test unhappy paths. + */ + const noUnallowedValues: Record = { + 'event.category': [], + 'event.kind': [], + 'event.outcome': [], + 'event.type': [], + }; + + /** + * Represents an index that has unallowed values, for the + * `event.category` field. The other fields in the collection, + * i.e. `event.kind`, don't have any unallowed values. + * + * This instance will be used to test paths where a field is + * NOT ECS complaint, because the index has unallowed values. + */ + const unallowedValues: Record = { + 'event.category': [ + { + count: 2, + fieldName: 'an_invalid_category', + }, + { + count: 1, + fieldName: 'theory', + }, + ], + 'event.kind': [], + 'event.outcome': [], + 'event.type': [], + }; + + /** + * This instance of a `FieldType` has the correct mapping for the + * `event.category` field. + * + * This instance will be used to test paths where the index has + * a valid mapping for the `event.category` field. + */ + const fieldMetadataCorrectMappingType: FieldType = { + field: 'event.category', + type: 'keyword', // <-- this index has the correct mapping type + }; + + /** + * This `EnrichedFieldMetadata` for the `event.category` field, + * represents a happy path result, where the index being checked: + * + * 1) The `type` of the field in the index, `keyword`, matches the expected + * `type` of the `event.category` field, as defined by the `EcsMetadata` + * 2) The index doesn't have any unallowed values for the `event.category` field + * + * Since this is a happy path result, it has the following values: + * `indexInvalidValues` is an empty array, because the index does not contain any invalid values + * `isEcsCompliant` is true, because the index has the expected mapping type, and no unallowed values + */ + const happyPathResultSample: EnrichedFieldMetadata = { + dashed_name: 'event-category', + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + example: 'authentication', + flat_name: 'event.category', + ignore_above: 1024, + level: 'core', + name: 'category', + normalize: ['array'], + short: 'Event category. The second categorization field in the hierarchy.', + type: 'keyword', + indexFieldName: 'event.category', + indexFieldType: 'keyword', // a valid mapping, because the `type` property from the `ecsMetadata` is also `keyword` + indexInvalidValues: [], // empty array, because the index does not contain any invalid values + hasEcsMetadata: true, + isEcsCompliant: true, // because the index has the expected mapping type, and no unallowed values + isInSameFamily: false, + }; + + /** + * Creates expected result matcher based on the happy path result sample. Please, add similar `expect` based assertions to it if anything breaks + * with an ECS upgrade, instead of hardcoding the values. + */ + const expectedResult = (extraFields: Record = {}) => + expect.objectContaining({ + ...happyPathResultSample, + ...extraFields, + allowed_values: expect.arrayContaining([ + expect.objectContaining({ + description: expect.any(String), + name: expect.any(String), + expected_event_types: expect.any(Array), + }), + ]), + }); + + test('it returns the happy path result when the index has no mapping conflicts, and no unallowed values', () => { + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index + unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index + }) + ).toEqual(expectedResult()); + }); + + test('it returns the happy path result when the index has no mapping conflicts, and the unallowedValues map does not contain an entry for the field', () => { + // create an `unallowedValues` that doesn't have an entry for `event.category`: + const noEntryForEventCategory: Record = omit( + 'event.category', + unallowedValues + ); + + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index + unallowedValues: noEntryForEventCategory, // a lookup in this map for the `event.category` field will return undefined + }) + ).toEqual(expectedResult()); + }); + + test('it returns a result with the expected `indexInvalidValues` and `isEcsCompliant` when the index has no mapping conflict, but it has unallowed values', () => { + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index + unallowedValues, // this index has unallowed values for the event.category field + }) + ).toEqual( + expectedResult({ + indexInvalidValues: [ + { + count: 2, + fieldName: 'an_invalid_category', + }, + { + count: 1, + fieldName: 'theory', + }, + ], + isEcsCompliant: false, // because there are unallowed values + }) + ); + }); + + test('it returns a result with the expected `isEcsCompliant` and `isInSameFamily` when the index type does not match ECS, but NO unallowed values', () => { + const indexFieldType = 'text'; + + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: { + field: 'event.category', // `event.category` is a `keyword`, per the ECS spec + type: indexFieldType, // this index has a mapping of `text` instead + }, + unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index + }) + ).toEqual( + expectedResult({ + indexFieldType, + isEcsCompliant: false, // `keyword` !== `text` + isInSameFamily: false, // `keyword` and `text` are not in the same family + }) + ); + }); + + test('it returns a result with the expected `isEcsCompliant` and `isInSameFamily` when the mapping is is in the same family', () => { + const indexFieldType = 'wildcard'; + + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: { + field: 'event.category', // `event.category` is a `keyword` per the ECS spec + type: indexFieldType, // this index has a mapping of `wildcard` instead + }, + unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index + }) + ).toEqual( + expectedResult({ + indexFieldType, + isEcsCompliant: false, // `wildcard` !== `keyword` + isInSameFamily: true, // `wildcard` and `keyword` are in the same family + }) + ); + }); + + test('it returns a result with the expected `indexInvalidValues`,`isEcsCompliant`, and `isInSameFamily` when the index has BOTH mapping conflicts, and unallowed values', () => { + const indexFieldType = 'text'; + + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: { + field: 'event.category', // `event.category` is a `keyword` per the ECS spec + type: indexFieldType, // this index has a mapping of `text` instead + }, + unallowedValues, // this index also has unallowed values for the event.category field + }) + ).toEqual( + expectedResult({ + indexFieldType, + indexInvalidValues: [ + { + count: 2, + fieldName: 'an_invalid_category', + }, + { + count: 1, + fieldName: 'theory', + }, + ], + isEcsCompliant: false, // because there are BOTH mapping conflicts and unallowed values + isInSameFamily: false, // `text` and `keyword` are not in the same family + }) + ); + }); + + test('it returns the expected result for a custom field, i.e. a field that does NOT have an entry in `ecsMetadata`', () => { + const field = 'a_custom_field'; // not defined by ECS + const indexFieldType = 'keyword'; + + expect( + getEnrichedFieldMetadata({ + ecsMetadata: EcsFlatTyped, + fieldMetadata: { + field, + type: indexFieldType, // no mapping conflict, because ECS doesn't define this field + }, + unallowedValues: noUnallowedValues, // no unallowed values for `a_custom_field` in this index + }) + ).toEqual({ + indexFieldName: field, + indexFieldType, + indexInvalidValues: [], + hasEcsMetadata: false, + isEcsCompliant: false, + isInSameFamily: false, // custom fields are never in the same family + }); + }); +}); + +describe('getMissingTimestampFieldMetadata', () => { + test('it returns the expected `EnrichedFieldMetadata`', () => { + expect(getMissingTimestampFieldMetadata()).toEqual({ + ...EcsFlatTyped['@timestamp'], + hasEcsMetadata: true, + indexFieldName: '@timestamp', + indexFieldType: '-', // the index did NOT define a mapping for @timestamp + indexInvalidValues: [], + isEcsCompliant: false, // an index must define the @timestamp mapping + isInSameFamily: false, // `date` is not a member of any families + }); + }); +}); + +describe('getPartitionedFieldMetadata', () => { + test('it places all the `EnrichedFieldMetadata` in the expected categories', () => { + const enrichedFieldMetadata: EnrichedFieldMetadata[] = [ + timestamp, + eventCategoryWithUnallowedValues, + hostNameWithTextMapping, + hostNameKeyword, + someField, + someFieldKeyword, + sourceIpWithTextMapping, + sourceIpKeyword, + sourcePort, + ]; + const expected: PartitionedFieldMetadata = { + all: [ + timestamp, + eventCategoryWithUnallowedValues, + hostNameWithTextMapping, + hostNameKeyword, + someField, + someFieldKeyword, + sourceIpWithTextMapping, + sourceIpKeyword, + sourcePort, + ], + ecsCompliant: [timestamp, sourcePort], + custom: [hostNameKeyword, someField, someFieldKeyword, sourceIpKeyword], + incompatible: [ + eventCategoryWithUnallowedValues, + hostNameWithTextMapping, + sourceIpWithTextMapping, + ], + sameFamily: [], + }; + + expect(getPartitionedFieldMetadata(enrichedFieldMetadata)).toEqual(expected); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.ts new file mode 100644 index 0000000000000..87adf1e2314e1 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { has } from 'lodash/fp'; + +import { EcsFlatTyped } from '../../../../../../constants'; +import { + EcsBasedFieldMetadata, + EnrichedFieldMetadata, + PartitionedFieldMetadata, + UnallowedValueCount, +} from '../../../../../../types'; +import { getIsInSameFamily } from './get_is_in_same_family'; + +export const getPartitionedFieldMetadata = ( + enrichedFieldMetadata: EnrichedFieldMetadata[] +): PartitionedFieldMetadata => + enrichedFieldMetadata.reduce( + (acc, x) => ({ + all: [...acc.all, x], + ecsCompliant: x.isEcsCompliant ? [...acc.ecsCompliant, x] : acc.ecsCompliant, + custom: !x.hasEcsMetadata ? [...acc.custom, x] : acc.custom, + incompatible: + x.hasEcsMetadata && !x.isEcsCompliant && !x.isInSameFamily + ? [...acc.incompatible, x] + : acc.incompatible, + sameFamily: x.isInSameFamily ? [...acc.sameFamily, x] : acc.sameFamily, + }), + { + all: [], + ecsCompliant: [], + custom: [], + incompatible: [], + sameFamily: [], + } + ); + +export interface FieldType { + field: string; + type: string; +} + +function shouldReadKeys(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +const getNextPathWithoutProperties = ({ + key, + pathWithoutProperties, + value, +}: { + key: string; + pathWithoutProperties: string; + value: unknown; +}): string => { + if (!pathWithoutProperties) { + return key; + } + + if (shouldReadKeys(value) && (key === 'properties' || key === 'fields')) { + return `${pathWithoutProperties}`; + } else { + return `${pathWithoutProperties}.${key}`; + } +}; + +export function getFieldTypes(mappingsProperties: Record): FieldType[] { + if (!shouldReadKeys(mappingsProperties)) { + throw new TypeError(`Root value is not flatten-able, received ${mappingsProperties}`); + } + + const result: FieldType[] = []; + (function flatten(prefix, object, pathWithoutProperties) { + for (const [key, value] of Object.entries(object)) { + const path = prefix ? `${prefix}.${key}` : key; + + const nextPathWithoutProperties = getNextPathWithoutProperties({ + key, + pathWithoutProperties, + value, + }); + + if (shouldReadKeys(value)) { + flatten(path, value, nextPathWithoutProperties); + } else { + if (nextPathWithoutProperties.endsWith('.type')) { + const pathWithoutType = nextPathWithoutProperties.slice( + 0, + nextPathWithoutProperties.lastIndexOf('.type') + ); + + result.push({ + field: pathWithoutType, + type: `${value}`, + }); + } + } + } + })('', mappingsProperties, ''); + + return result; +} + +export const isMappingCompatible = ({ + ecsExpectedType, + type, +}: { + ecsExpectedType: string | undefined; + type: string; +}): boolean => type === ecsExpectedType; + +export const getEnrichedFieldMetadata = ({ + ecsMetadata, + fieldMetadata, + unallowedValues, +}: { + ecsMetadata: EcsFlatTyped; + fieldMetadata: FieldType; + unallowedValues: Record; +}): EnrichedFieldMetadata => { + const { field, type } = fieldMetadata; + const indexInvalidValues = unallowedValues[field] ?? []; + + if (has(fieldMetadata.field, ecsMetadata)) { + const ecsExpectedType = ecsMetadata[field].type; + const isEcsCompliant = + isMappingCompatible({ ecsExpectedType, type }) && indexInvalidValues.length === 0; + + const isInSameFamily = + !isMappingCompatible({ ecsExpectedType, type }) && + indexInvalidValues.length === 0 && + getIsInSameFamily({ ecsExpectedType, type }); + + return { + ...ecsMetadata[field], + indexFieldName: field, + indexFieldType: type, + indexInvalidValues, + hasEcsMetadata: true, + isEcsCompliant, + isInSameFamily, + }; + } else { + return { + indexFieldName: field, + indexFieldType: type, + indexInvalidValues: [], + hasEcsMetadata: false, + isEcsCompliant: false, + isInSameFamily: false, // custom fields are never in the same family + }; + } +}; + +export const getMissingTimestampFieldMetadata = (): EcsBasedFieldMetadata => ({ + ...EcsFlatTyped['@timestamp'], + hasEcsMetadata: true, + indexFieldName: '@timestamp', + indexFieldType: '-', + indexInvalidValues: [], + isEcsCompliant: false, + isInSameFamily: false, // `date` is not a member of any families +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/index.tsx index e755dbd2df8f1..c1d1ead5e7891 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/index.tsx @@ -9,8 +9,8 @@ import { EuiBadge, EuiFlexGroup, EuiFlexItem, EuiToolTip } from '@elastic/eui'; import React from 'react'; import styled from 'styled-components'; -import { getPatternIlmPhaseDescription } from '../../../../../../helpers'; import type { IlmExplainPhaseCounts, IlmPhase } from '../../../../../../types'; +import { getPatternIlmPhaseDescription } from './utils/get_pattern_ilm_phase_description'; const PhaseCountsFlexGroup = styled(EuiFlexGroup)` display: inline-flex; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/translations.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/translations.ts new file mode 100644 index 0000000000000..d6a87384c9ae4 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/translations.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const UNMANAGED_PATTERN_TOOLTIP = ({ + indices, + pattern, +}: { + indices: number; + pattern: string; +}) => + i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.unmanagedPatternTooltip', { + values: { indices, pattern }, + defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} unmanaged by Index Lifecycle Management (ILM)`, + }); + +export const WARM_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) => + i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.warmPatternTooltip', { + values: { indices, pattern }, + defaultMessage: + '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} warm. Warm indices are no longer being updated but are still being queried.', + }); + +export const HOT_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) => + i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.hotPatternTooltip', { + values: { indices, pattern }, + defaultMessage: + '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} hot. Hot indices are actively being updated and queried.', + }); + +export const FROZEN_PATTERN_TOOLTIP = ({ + indices, + pattern, +}: { + indices: number; + pattern: string; +}) => + i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.frozenPatternTooltip', { + values: { indices, pattern }, + defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.`, + }); + +export const COLD_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) => + i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.coldPatternTooltip', { + values: { indices, pattern }, + defaultMessage: + '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.', + }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.test.ts new file mode 100644 index 0000000000000..3faff162dbffa --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getPatternIlmPhaseDescription } from './get_pattern_ilm_phase_description'; + +describe('getPatternIlmPhaseDescription', () => { + const phases: Array<{ + expected: string; + indices: number; + pattern: string; + phase: string; + }> = [ + { + expected: + '1 index matching the .alerts-security.alerts-default pattern is hot. Hot indices are actively being updated and queried.', + indices: 1, + pattern: '.alerts-security.alerts-default', + phase: 'hot', + }, + { + expected: + '2 indices matching the .alerts-security.alerts-default pattern are hot. Hot indices are actively being updated and queried.', + indices: 2, + pattern: '.alerts-security.alerts-default', + phase: 'hot', + }, + { + expected: + '1 index matching the .alerts-security.alerts-default pattern is warm. Warm indices are no longer being updated but are still being queried.', + indices: 1, + pattern: '.alerts-security.alerts-default', + phase: 'warm', + }, + { + expected: + '2 indices matching the .alerts-security.alerts-default pattern are warm. Warm indices are no longer being updated but are still being queried.', + indices: 2, + pattern: '.alerts-security.alerts-default', + phase: 'warm', + }, + { + expected: + '1 index matching the .alerts-security.alerts-default pattern is cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.', + indices: 1, + pattern: '.alerts-security.alerts-default', + phase: 'cold', + }, + { + expected: + '2 indices matching the .alerts-security.alerts-default pattern are cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.', + indices: 2, + pattern: '.alerts-security.alerts-default', + phase: 'cold', + }, + { + expected: + "1 index matching the .alerts-security.alerts-default pattern is frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.", + indices: 1, + pattern: '.alerts-security.alerts-default', + phase: 'frozen', + }, + { + expected: + "2 indices matching the .alerts-security.alerts-default pattern are frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.", + indices: 2, + pattern: '.alerts-security.alerts-default', + phase: 'frozen', + }, + { + expected: + '1 index matching the .alerts-security.alerts-default pattern is unmanaged by Index Lifecycle Management (ILM)', + indices: 1, + pattern: '.alerts-security.alerts-default', + phase: 'unmanaged', + }, + { + expected: + '2 indices matching the .alerts-security.alerts-default pattern are unmanaged by Index Lifecycle Management (ILM)', + indices: 2, + pattern: '.alerts-security.alerts-default', + phase: 'unmanaged', + }, + { + expected: '', + indices: 1, + pattern: '.alerts-security.alerts-default', + phase: 'some-other-phase', + }, + { + expected: '', + indices: 2, + pattern: '.alerts-security.alerts-default', + phase: 'some-other-phase', + }, + ]; + + phases.forEach(({ expected, indices, pattern, phase }) => { + test(`it returns the expected description when indices is ${indices}, pattern is ${pattern}, and phase is ${phase}`, () => { + expect(getPatternIlmPhaseDescription({ indices, pattern, phase })).toBe(expected); + }); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.ts new file mode 100644 index 0000000000000..20adb0c0c5bf9 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as i18n from '../translations'; + +export const getPatternIlmPhaseDescription = ({ + indices, + pattern, + phase, +}: { + indices: number; + pattern: string; + phase: string; +}): string => { + switch (phase) { + case 'hot': + return i18n.HOT_PATTERN_TOOLTIP({ indices, pattern }); + case 'warm': + return i18n.WARM_PATTERN_TOOLTIP({ indices, pattern }); + case 'cold': + return i18n.COLD_PATTERN_TOOLTIP({ indices, pattern }); + case 'frozen': + return i18n.FROZEN_PATTERN_TOOLTIP({ indices, pattern }); + case 'unmanaged': + return i18n.UNMANAGED_PATTERN_TOOLTIP({ indices, pattern }); + default: + return ''; + } +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.test.tsx index 022119e4613e5..1efaa01d36f9e 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.test.tsx @@ -17,7 +17,7 @@ import { omit } from 'lodash/fp'; import React from 'react'; import { TestExternalProviders } from '../../../../mock/test_providers/test_providers'; -import { EMPTY_STAT } from '../../../../helpers'; +import { EMPTY_STAT } from '../../../../constants'; import { getDocsCountPercent, getIncompatibleStatColor, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.tsx index b41b5604d1b98..327c5b5e46667 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.tsx @@ -18,7 +18,8 @@ import moment from 'moment'; import styled from 'styled-components'; import { euiThemeVars } from '@kbn/ui-theme'; -import { EMPTY_STAT, getIlmPhaseDescription } from '../../../../helpers'; +import { EMPTY_STAT } from '../../../../constants'; +import { getIlmPhaseDescription } from '../../../../utils/get_ilm_phase_description'; import { INCOMPATIBLE_INDEX_TOOL_TIP } from '../../../../stat_label/translations'; import { INDEX_SIZE_TOOLTIP } from '../../../../translations'; import * as i18n from './translations'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/index.test.tsx index 5863649335edb..01d45a3784a5f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/index.test.tsx @@ -9,7 +9,7 @@ import numeral from '@elastic/numeral'; import { render, screen } from '@testing-library/react'; import React from 'react'; -import { EMPTY_STAT } from '../../../../helpers'; +import { EMPTY_STAT } from '../../../../constants'; import { getSummaryTableColumns } from './helpers'; import { mockIlmExplain } from '../../../../mock/ilm_explain/mock_ilm_explain'; import { auditbeatWithAllResults } from '../../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.test.ts new file mode 100644 index 0000000000000..306c9f93d83b6 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.test.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { omit } from 'lodash/fp'; + +import { mockStatsAuditbeatIndex } from '../../../../mock/stats/mock_stats_packetbeat_index'; +import { mockStatsPacketbeatIndex } from '../../../../mock/stats/mock_stats_auditbeat_index'; +import { mockIlmExplain } from '../../../../mock/ilm_explain/mock_ilm_explain'; +import { mockStats } from '../../../../mock/stats/mock_stats'; +import { getIndexNames, getPatternDocsCount, getPatternSizeInBytes } from './stats'; +import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; + +describe('getIndexNames', () => { + const isILMAvailable = true; + const ilmPhases = ['hot', 'warm', 'unmanaged']; + + test('returns the expected index names when they have an ILM phase included in the ilmPhases list', () => { + expect( + getIndexNames({ + ilmExplain: mockIlmExplain, // <-- the mock indexes have 'hot' ILM phases + ilmPhases, + isILMAvailable, + stats: mockStats, + }) + ).toEqual([ + '.ds-packetbeat-8.6.1-2023.02.04-000001', + '.ds-packetbeat-8.5.3-2023.02.04-000001', + 'auditbeat-custom-index-1', + ]); + }); + + test('returns the expected filtered index names when they do NOT have an ILM phase included in the ilmPhases list', () => { + expect( + getIndexNames({ + ilmExplain: mockIlmExplain, // <-- the mock indexes have 'hot' and 'unmanaged' ILM phases... + ilmPhases: ['warm', 'unmanaged'], // <-- ...but we don't ask for 'hot' + isILMAvailable, + stats: mockStats, + }) + ).toEqual(['auditbeat-custom-index-1']); // <-- the 'unmanaged' index + }); + + test('returns the expected index names when the `ilmExplain` is missing a record for an index', () => { + // the following `ilmExplain` is missing a record for one of the two packetbeat indexes: + const ilmExplainWithMissingIndex: Record = omit( + '.ds-packetbeat-8.6.1-2023.02.04-000001', + mockIlmExplain + ); + + expect( + getIndexNames({ + ilmExplain: ilmExplainWithMissingIndex, // <-- the mock indexes have 'hot' ILM phases... + ilmPhases: ['hot', 'warm', 'unmanaged'], + isILMAvailable, + stats: mockStats, + }) + ).toEqual(['.ds-packetbeat-8.5.3-2023.02.04-000001', 'auditbeat-custom-index-1']); // <-- only includes two of the three indices, because the other one is missing an ILM explain record + }); + + test('returns empty index names when `ilmPhases` is empty', () => { + expect( + getIndexNames({ + ilmExplain: mockIlmExplain, + ilmPhases: [], + isILMAvailable, + stats: mockStats, + }) + ).toEqual([]); + }); + + test('returns empty index names when they have an ILM phase that matches', () => { + expect( + getIndexNames({ + ilmExplain: null, + ilmPhases, + isILMAvailable, + stats: mockStats, + }) + ).toEqual([]); + }); + + test('returns empty index names when just `stats` is null', () => { + expect( + getIndexNames({ + ilmExplain: mockIlmExplain, + ilmPhases, + isILMAvailable, + stats: null, + }) + ).toEqual([]); + }); + + test('returns empty index names when both `ilmExplain` and `stats` are null', () => { + expect( + getIndexNames({ + ilmExplain: null, + ilmPhases, + isILMAvailable, + stats: null, + }) + ).toEqual([]); + }); +}); + +describe('getPatternDocsCount', () => { + test('it returns the expected total given a subset of index names in the stats', () => { + const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001'; + const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs; + + expect( + getPatternDocsCount({ + indexNames: [indexName], + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(expectedCount); + }); + + test('it returns the expected total given all index names in the stats', () => { + const allIndexNamesInStats = [ + '.ds-packetbeat-8.6.1-2023.02.04-000001', + '.ds-packetbeat-8.5.3-2023.02.04-000001', + ]; + + expect( + getPatternDocsCount({ + indexNames: allIndexNamesInStats, + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(3258632); + }); + + test('it returns zero given an empty collection of index names', () => { + expect( + getPatternDocsCount({ + indexNames: [], // <-- empty + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(0); + }); + + test('it returns the expected total for a green index', () => { + const indexName = 'auditbeat-custom-index-1'; + const expectedCount = mockStatsAuditbeatIndex[indexName].num_docs; + + expect( + getPatternDocsCount({ + indexNames: [indexName], + stats: mockStatsAuditbeatIndex, + }) + ).toEqual(expectedCount); + }); +}); + +describe('getPatternSizeInBytes', () => { + test('it returns the expected total given a subset of index names in the stats', () => { + const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001'; + const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes; + + expect( + getPatternSizeInBytes({ + indexNames: [indexName], + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(expectedCount); + }); + + test('it returns the expected total given all index names in the stats', () => { + const allIndexNamesInStats = [ + '.ds-packetbeat-8.6.1-2023.02.04-000001', + '.ds-packetbeat-8.5.3-2023.02.04-000001', + ]; + + expect( + getPatternSizeInBytes({ + indexNames: allIndexNamesInStats, + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(1464758182); + }); + + test('it returns undefined given an empty collection of index names', () => { + expect( + getPatternSizeInBytes({ + indexNames: [], // <-- empty + stats: mockStatsPacketbeatIndex, + }) + ).toBeUndefined(); + }); + + test('it returns undefined if sizeInByte in not an integer', () => { + const indexName = 'auditbeat-custom-index-1'; + + expect( + getPatternSizeInBytes({ + indexNames: [indexName], + stats: { [indexName]: { ...mockStatsAuditbeatIndex[indexName], size_in_bytes: null } }, + }) + ).toBeUndefined(); + }); + + test('it returns the expected total for an index', () => { + const indexName = 'auditbeat-custom-index-1'; + const expectedCount = mockStatsAuditbeatIndex[indexName].size_in_bytes; + + expect( + getPatternSizeInBytes({ + indexNames: [indexName], + stats: mockStatsAuditbeatIndex, + }) + ).toEqual(expectedCount); + }); + + test('it returns the expected total for indices', () => { + const expectedCount = Object.values(mockStatsPacketbeatIndex).reduce( + (acc, { size_in_bytes: sizeInBytes }) => { + return acc + (sizeInBytes ?? 0); + }, + 0 + ); + + expect( + getPatternSizeInBytes({ + indexNames: [ + '.ds-packetbeat-8.6.1-2023.02.04-000001', + '.ds-packetbeat-8.5.3-2023.02.04-000001', + ], + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(expectedCount); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.ts new file mode 100644 index 0000000000000..83e2b592dc079 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; + +import { getDocsCount, getSizeInBytes } from '../../../../utils/stats'; +import { MeteringStatsIndex } from '../../../../types'; +import { getIlmPhase } from '../helpers'; + +export const getPatternDocsCount = ({ + indexNames, + stats, +}: { + indexNames: string[]; + stats: Record | null; +}): number => + indexNames.reduce( + (acc: number, indexName: string) => acc + getDocsCount({ stats, indexName }), + 0 + ); + +export const getPatternSizeInBytes = ({ + indexNames, + stats, +}: { + indexNames: string[]; + stats: Record | null; +}): number | undefined => { + let sum; + for (let i = 0; i < indexNames.length; i++) { + const currentSizeInBytes = getSizeInBytes({ stats, indexName: indexNames[i] }); + if (currentSizeInBytes != null) { + if (sum == null) { + sum = 0; + } + sum += currentSizeInBytes; + } else { + return undefined; + } + } + return sum; +}; + +const EMPTY_INDEX_NAMES: string[] = []; +export const getIndexNames = ({ + ilmExplain, + ilmPhases, + isILMAvailable, + stats, +}: { + ilmExplain: Record | null; + ilmPhases: string[]; + isILMAvailable: boolean; + stats: Record | null; +}): string[] => { + if (((isILMAvailable && ilmExplain != null) || !isILMAvailable) && stats != null) { + const allIndexNames = Object.keys(stats); + const filteredByIlmPhase = isILMAvailable + ? allIndexNames.filter((indexName) => + ilmPhases.includes(getIlmPhase(ilmExplain?.[indexName], isILMAvailable) ?? '') + ) + : allIndexNames; + + return filteredByIlmPhase; + } else { + return EMPTY_INDEX_NAMES; + } +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.test.ts index e4091d38c20d9..da46cd2b40f33 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.test.ts @@ -8,7 +8,7 @@ import numeral from '@elastic/numeral'; import { euiThemeVars } from '@kbn/ui-theme'; -import { EMPTY_STAT } from '../../helpers'; +import { EMPTY_STAT } from '../../constants'; import { DEFAULT_INDEX_COLOR, getFillColor, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.ts index 456752eef627e..283854a62acf2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.ts @@ -9,9 +9,9 @@ import type { Datum, Key, ArrayNode } from '@elastic/charts'; import { euiThemeVars } from '@kbn/ui-theme'; import { orderBy } from 'lodash/fp'; -import { getDocsCount, getSizeInBytes } from '../../helpers'; import { getIlmPhase } from '../indices_details/pattern/helpers'; import { PatternRollup } from '../../types'; +import { getDocsCount, getSizeInBytes } from '../../utils/stats'; export interface LegendItem { color: string | null; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/index.test.tsx index 7eff122a3df06..3ebb44a8306ef 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/index.test.tsx @@ -9,7 +9,7 @@ import numeral from '@elastic/numeral'; import { render, screen } from '@testing-library/react'; import React from 'react'; -import { EMPTY_STAT } from '../../helpers'; +import { EMPTY_STAT } from '../../constants'; import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup'; import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { packetbeatNoResults } from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/storage_treemap/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/storage_treemap/index.test.tsx index 6206dfca5742f..a8998a134416f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/storage_treemap/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/storage_treemap/index.test.tsx @@ -12,7 +12,7 @@ import userEvent from '@testing-library/user-event'; import React from 'react'; import { FlattenedBucket, getFlattenedBuckets, getLegendItems } from '../helpers'; -import { EMPTY_STAT } from '../../../helpers'; +import { EMPTY_STAT } from '../../../constants'; import { alertIndexWithAllResults } from '../../../mock/pattern_rollup/mock_alerts_pattern_rollup'; import { auditbeatWithAllResults } from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { packetbeatNoResults } from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/ilm_phase_filter/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/ilm_phase_filter/index.tsx index 0a52777c1ab93..cd7148d38b3aa 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/ilm_phase_filter/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/ilm_phase_filter/index.tsx @@ -15,7 +15,7 @@ import { import React, { useCallback, useMemo } from 'react'; import { ilmPhaseOptionsStatic } from '../../constants'; -import { getIlmPhaseDescription } from '../../helpers'; +import { getIlmPhaseDescription } from '../../utils/get_ilm_phase_description'; import { ILM_PHASE, INDEX_LIFECYCLE_MANAGEMENT_PHASES, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/index.test.tsx index d3861b46b3e12..322d30be6dd81 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/index.test.tsx @@ -9,7 +9,7 @@ import numeral from '@elastic/numeral'; import { render, screen } from '@testing-library/react'; import React from 'react'; -import { EMPTY_STAT } from '../helpers'; +import { EMPTY_STAT } from '../constants'; import { alertIndexWithAllResults } from '../mock/pattern_rollup/mock_alerts_pattern_rollup'; import { auditbeatWithAllResults } from '../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { packetbeatNoResults } from '../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; @@ -25,7 +25,7 @@ import { getTotalIndices, getTotalIndicesChecked, getTotalSizeInBytes, -} from '../hooks/use_results_rollup/helpers'; +} from '../hooks/use_results_rollup/utils/stats'; const defaultBytesFormat = '0,0.[0]b'; const formatBytes = (value: number | undefined) => diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts index fe453dd3a7370..dfbc0f69f82aa 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts @@ -7,8 +7,8 @@ import { orderBy } from 'lodash/fp'; -import { getDocsCount } from '../../../helpers'; import type { IndexToCheck, MeteringStatsIndex, PatternRollup } from '../../../types'; +import { getDocsCount } from '../../../utils/stats'; export const getIndexToCheck = ({ indexName, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/index.test.tsx index 5ebb61152b1ae..14368907362fb 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/index.test.tsx @@ -19,7 +19,7 @@ import { mockUnallowedValuesResponse } from '../../../mock/unallowed_values/mock import { CANCEL, CHECK_ALL } from '../../../translations'; import { OnCheckCompleted, UnallowedValueRequestItem } from '../../../types'; import { CheckAll } from '.'; -import { EMPTY_STAT } from '../../../helpers'; +import { EMPTY_STAT } from '../../../constants'; const defaultBytesFormat = '0,0.[0]b'; const mockFormatBytes = (value: number | undefined) => diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.test.tsx index fa813ccfaba71..1bcfc9ec12aa3 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.test.tsx @@ -10,7 +10,7 @@ import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; -import { EMPTY_STAT } from '../../helpers'; +import { EMPTY_STAT } from '../../constants'; import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup'; import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { packetbeatNoResults } from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; @@ -26,7 +26,7 @@ import { getTotalIndices, getTotalIndicesChecked, getTotalSizeInBytes, -} from '../../hooks/use_results_rollup/helpers'; +} from '../../hooks/use_results_rollup/utils/stats'; const mockCopyToClipboard = jest.fn((value) => true); jest.mock('@elastic/eui', () => { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.tsx index 71b6cbad9c171..d0037032172c3 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.tsx @@ -27,10 +27,11 @@ import { getSummaryTableItems, } from '../../data_quality_details/indices_details/pattern/helpers'; import type { DataQualityCheckResult, IndexToCheck, PatternRollup } from '../../types'; -import { getErrorSummaries, getSizeInBytes } from '../../helpers'; import { useDataQualityContext } from '../../data_quality_context'; import { useResultsRollupContext } from '../../contexts/results_rollup_context'; import { Actions } from '../../actions'; +import { getErrorSummaries } from './utils/get_error_summaries'; +import { getSizeInBytes } from '../../utils/stats'; const StyledActionsContainerFlexItem = styled(EuiFlexItem)` margin-top: auto; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.test.ts new file mode 100644 index 0000000000000..c1e78b92c0fd8 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.test.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertIndexNoResults } from '../../../mock/pattern_rollup/mock_alerts_pattern_rollup'; +import { + auditbeatNoResults, + auditbeatWithAllResults, +} from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; +import { + packetbeatNoResults, + packetbeatWithSomeErrors, +} from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; +import { DataQualityCheckResult, PatternRollup } from '../../../types'; +import { + getErrorSummaries, + getErrorSummariesForRollup, + getErrorSummary, +} from './get_error_summaries'; + +describe('getErrorSummary', () => { + test('it returns the expected error summary', () => { + const resultWithError: DataQualityCheckResult = { + docsCount: 1630289, + error: + 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', + ilmPhase: 'hot', + incompatible: undefined, + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'packetbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }; + + expect(getErrorSummary(resultWithError)).toEqual({ + error: + 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + pattern: 'packetbeat-*', + }); + }); +}); + +describe('getErrorSummariesForRollup', () => { + test('it returns the expected array of `ErrorSummary` when the `PatternRollup` contains errors', () => { + expect(getErrorSummariesForRollup(packetbeatWithSomeErrors)).toEqual([ + { + error: + 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + pattern: 'packetbeat-*', + }, + ]); + }); + + test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` contains all results, with NO errors', () => { + expect(getErrorSummariesForRollup(auditbeatWithAllResults)).toEqual([]); + }); + + test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` has NO results', () => { + expect(getErrorSummariesForRollup(auditbeatNoResults)).toEqual([]); + }); + + test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` is undefined', () => { + expect(getErrorSummariesForRollup(undefined)).toEqual([]); + }); + + test('it returns BOTH the expected (root) pattern-level error, and an index-level error when `PatternRollup` has both', () => { + const withPatternLevelError: PatternRollup = { + ...packetbeatWithSomeErrors, + error: 'This is a pattern-level error', + }; + + expect(getErrorSummariesForRollup(withPatternLevelError)).toEqual([ + { + error: 'This is a pattern-level error', + indexName: null, + pattern: 'packetbeat-*', + }, + { + error: + 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + pattern: 'packetbeat-*', + }, + ]); + }); + + test('it returns the expected (root) pattern-level error when there are no index-level results', () => { + const withPatternLevelError: PatternRollup = { + ...auditbeatNoResults, + error: 'This is a pattern-level error', + }; + + expect(getErrorSummariesForRollup(withPatternLevelError)).toEqual([ + { + error: 'This is a pattern-level error', + indexName: null, + pattern: 'auditbeat-*', + }, + ]); + }); +}); + +describe('getErrorSummaries', () => { + test('it returns an empty array when patternRollups is empty', () => { + expect(getErrorSummaries({})).toEqual([]); + }); + + test('it returns an empty array when none of the patternRollups have errors', () => { + expect( + getErrorSummaries({ + '.alerts-security.alerts-default': alertIndexNoResults, + 'auditbeat-*': auditbeatWithAllResults, + 'packetbeat-*': packetbeatNoResults, + }) + ).toEqual([]); + }); + + test('it returns the expected array of `ErrorSummary` when some of the `PatternRollup` contain errors', () => { + expect( + getErrorSummaries({ + '.alerts-security.alerts-default': alertIndexNoResults, + 'auditbeat-*': auditbeatWithAllResults, + 'packetbeat-*': packetbeatWithSomeErrors, // <-- has errors + }) + ).toEqual([ + { + error: + 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + pattern: 'packetbeat-*', + }, + ]); + }); + + test('it returns the expected array of `ErrorSummary` when there are both pattern-level and index-level errors', () => { + const withPatternLevelError: PatternRollup = { + ...auditbeatNoResults, + error: 'This is a pattern-level error', + }; + + expect( + getErrorSummaries({ + '.alerts-security.alerts-default': alertIndexNoResults, + 'auditbeat-*': withPatternLevelError, // <-- has pattern-level errors + 'packetbeat-*': packetbeatWithSomeErrors, // <-- has index-level errors + }) + ).toEqual([ + { + error: 'This is a pattern-level error', + indexName: null, + pattern: 'auditbeat-*', + }, + { + error: + 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + pattern: 'packetbeat-*', + }, + ]); + }); + + test('it returns the expected array of `ErrorSummary` when there are just pattern-level errors', () => { + const withPatternLevelError: PatternRollup = { + ...auditbeatNoResults, + error: 'This is a pattern-level error', + }; + + expect( + getErrorSummaries({ + '.alerts-security.alerts-default': alertIndexNoResults, + 'auditbeat-*': withPatternLevelError, // <-- has pattern-level errors + 'packetbeat-*': packetbeatNoResults, + }) + ).toEqual([ + { + error: 'This is a pattern-level error', + indexName: null, + pattern: 'auditbeat-*', + }, + ]); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.ts new file mode 100644 index 0000000000000..a3b6b8ac7d1d8 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import sortBy from 'lodash/fp/sortBy'; +import { DataQualityCheckResult, ErrorSummary, PatternRollup } from '../../../types'; + +export const getErrorSummary = ({ + error, + indexName, + pattern, +}: DataQualityCheckResult): ErrorSummary => ({ + error: String(error), + indexName, + pattern, +}); + +export const getErrorSummariesForRollup = ( + patternRollup: PatternRollup | undefined +): ErrorSummary[] => { + const maybePatternErrorSummary: ErrorSummary[] = + patternRollup != null && patternRollup.error != null + ? [{ pattern: patternRollup.pattern, indexName: null, error: patternRollup.error }] + : []; + + if (patternRollup != null && patternRollup.results != null) { + const unsortedResults: DataQualityCheckResult[] = Object.values(patternRollup.results); + const sortedResults = sortBy('indexName', unsortedResults); + + return sortedResults.reduce( + (acc, result) => [...acc, ...(result.error != null ? [getErrorSummary(result)] : [])], + maybePatternErrorSummary + ); + } else { + return maybePatternErrorSummary; + } +}; + +export const getErrorSummaries = ( + patternRollups: Record +): ErrorSummary[] => { + const allPatterns: string[] = Object.keys(patternRollups); + + // sort the patterns A-Z: + const sortedPatterns = [...allPatterns].sort((a, b) => { + return a.localeCompare(b); + }); + + return sortedPatterns.reduce( + (acc, pattern) => [...acc, ...getErrorSummariesForRollup(patternRollups[pattern])], + [] + ); +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.test.ts deleted file mode 100644 index 0058e2436a526..0000000000000 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.test.ts +++ /dev/null @@ -1,1537 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; -import { omit } from 'lodash/fp'; - -import { - FieldType, - getDocsCount, - getErrorSummary, - getErrorSummaries, - getErrorSummariesForRollup, - getEnrichedFieldMetadata, - getFieldTypes, - getPatternIlmPhaseDescription, - getIlmPhaseDescription, - getIndexNames, - getIsInSameFamily, - getMissingTimestampFieldMetadata, - getPartitionedFieldMetadata, - getPartitionedFieldMetadataStats, - getSizeInBytes, - getTotalDocsCount, - getTotalPatternIncompatible, - getTotalPatternIndicesChecked, - getTotalPatternSameFamily, - getTotalSizeInBytes, - isMappingCompatible, - postStorageResult, - getStorageResults, - StorageResult, - formatStorageResult, -} from './helpers'; -import { - hostNameWithTextMapping, - hostNameKeyword, - someField, - someFieldKeyword, - sourceIpWithTextMapping, - sourceIpKeyword, - sourcePort, - timestamp, - eventCategoryWithUnallowedValues, -} from './mock/enriched_field_metadata/mock_enriched_field_metadata'; -import { mockIlmExplain } from './mock/ilm_explain/mock_ilm_explain'; -import { mockMappingsProperties } from './mock/mappings_properties/mock_mappings_properties'; -import { alertIndexNoResults } from './mock/pattern_rollup/mock_alerts_pattern_rollup'; -import { - packetbeatNoResults, - packetbeatWithSomeErrors, -} from './mock/pattern_rollup/mock_packetbeat_pattern_rollup'; -import { - auditbeatNoResults, - auditbeatWithAllResults, -} from './mock/pattern_rollup/mock_auditbeat_pattern_rollup'; -import { mockStats } from './mock/stats/mock_stats'; -import { mockStatsAuditbeatIndex } from './mock/stats/mock_stats_packetbeat_index'; -import { mockStatsPacketbeatIndex } from './mock/stats/mock_stats_auditbeat_index'; -import { - COLD_DESCRIPTION, - FROZEN_DESCRIPTION, - HOT_DESCRIPTION, - UNMANAGED_DESCRIPTION, - WARM_DESCRIPTION, -} from './translations'; -import { - DataQualityCheckResult, - EnrichedFieldMetadata, - PartitionedFieldMetadata, - PatternRollup, - UnallowedValueCount, -} from './types'; -import { httpServiceMock } from '@kbn/core-http-browser-mocks'; -import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; -import { EcsFlatTyped } from './constants'; -import { mockPartitionedFieldMetadataWithSameFamily } from './mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family'; - -describe('helpers', () => { - describe('getTotalPatternSameFamily', () => { - const baseResult: DataQualityCheckResult = { - docsCount: 4, - error: null, - ilmPhase: 'unmanaged', - incompatible: 3, - indexName: 'auditbeat-custom-index-1', - markdownComments: [ - '### auditbeat-custom-index-1\n', - '| Result | Index | Docs | Incompatible fields | ILM Phase |\n|--------|-------|------|---------------------|-----------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | `unmanaged` |\n\n', - '### **Incompatible fields** `3` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n', - "#### 3 incompatible fields, 0 fields with mappings in the same family\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\nIncompatible fields with mappings in the same family have exactly the same search behavior but may have different space usage or performance characteristics.\n\nWhen an incompatible field is not in the same family:\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", - '\n#### Incompatible field mappings - auditbeat-custom-index-1\n\n\n| Field | ECS mapping type (expected) | Index mapping type (actual) | \n|-------|-----------------------------|-----------------------------|\n| host.name | `keyword` | `text` |\n| source.ip | `ip` | `text` |\n\n#### Incompatible field values - auditbeat-custom-index-1\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `an_invalid_category` (2),\n`theory` (1) |\n\n', - ], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }; - - it('returns undefined when results is undefined', () => { - expect(getTotalPatternSameFamily(undefined)).toBeUndefined(); - }); - - it('returns 0 when results is an empty object', () => { - expect(getTotalPatternSameFamily({})).toBe(0); - }); - - it('should sum sameFamily values and return the total', () => { - const results: Record = { - a: { - ...baseResult, - indexName: 'a', - markdownComments: [], - pattern: 'pattern', - sameFamily: 2, - }, - b: { - ...baseResult, - indexName: 'b', - markdownComments: [], - pattern: 'pattern', - sameFamily: 3, - }, - c: { ...baseResult, indexName: 'c', markdownComments: [], pattern: 'pattern' }, - }; - - expect(getTotalPatternSameFamily(results)).toBe(5); - }); - - it('handles a mix of defined and undefined sameFamily values', () => { - const results: Record = { - a: { - ...baseResult, - indexName: 'a', - markdownComments: [], - pattern: 'pattern', - sameFamily: 1, - }, - b: { - ...baseResult, - indexName: 'b', - markdownComments: [], - pattern: 'pattern', - sameFamily: undefined, - }, - c: { - ...baseResult, - indexName: 'c', - markdownComments: [], - pattern: 'pattern', - sameFamily: 2, - }, - }; - - expect(getTotalPatternSameFamily(results)).toBe(3); - }); - }); - - describe('getIndexNames', () => { - const isILMAvailable = true; - const ilmPhases = ['hot', 'warm', 'unmanaged']; - - test('returns the expected index names when they have an ILM phase included in the ilmPhases list', () => { - expect( - getIndexNames({ - ilmExplain: mockIlmExplain, // <-- the mock indexes have 'hot' ILM phases - ilmPhases, - isILMAvailable, - stats: mockStats, - }) - ).toEqual([ - '.ds-packetbeat-8.6.1-2023.02.04-000001', - '.ds-packetbeat-8.5.3-2023.02.04-000001', - 'auditbeat-custom-index-1', - ]); - }); - - test('returns the expected filtered index names when they do NOT have an ILM phase included in the ilmPhases list', () => { - expect( - getIndexNames({ - ilmExplain: mockIlmExplain, // <-- the mock indexes have 'hot' and 'unmanaged' ILM phases... - ilmPhases: ['warm', 'unmanaged'], // <-- ...but we don't ask for 'hot' - isILMAvailable, - stats: mockStats, - }) - ).toEqual(['auditbeat-custom-index-1']); // <-- the 'unmanaged' index - }); - - test('returns the expected index names when the `ilmExplain` is missing a record for an index', () => { - // the following `ilmExplain` is missing a record for one of the two packetbeat indexes: - const ilmExplainWithMissingIndex: Record = omit( - '.ds-packetbeat-8.6.1-2023.02.04-000001', - mockIlmExplain - ); - - expect( - getIndexNames({ - ilmExplain: ilmExplainWithMissingIndex, // <-- the mock indexes have 'hot' ILM phases... - ilmPhases: ['hot', 'warm', 'unmanaged'], - isILMAvailable, - stats: mockStats, - }) - ).toEqual(['.ds-packetbeat-8.5.3-2023.02.04-000001', 'auditbeat-custom-index-1']); // <-- only includes two of the three indices, because the other one is missing an ILM explain record - }); - - test('returns empty index names when `ilmPhases` is empty', () => { - expect( - getIndexNames({ - ilmExplain: mockIlmExplain, - ilmPhases: [], - isILMAvailable, - stats: mockStats, - }) - ).toEqual([]); - }); - - test('returns empty index names when they have an ILM phase that matches', () => { - expect( - getIndexNames({ - ilmExplain: null, - ilmPhases, - isILMAvailable, - stats: mockStats, - }) - ).toEqual([]); - }); - - test('returns empty index names when just `stats` is null', () => { - expect( - getIndexNames({ - ilmExplain: mockIlmExplain, - ilmPhases, - isILMAvailable, - stats: null, - }) - ).toEqual([]); - }); - - test('returns empty index names when both `ilmExplain` and `stats` are null', () => { - expect( - getIndexNames({ - ilmExplain: null, - ilmPhases, - isILMAvailable, - stats: null, - }) - ).toEqual([]); - }); - }); - - describe('getFieldTypes', () => { - const expected = [ - { - field: '@timestamp', - type: 'date', - }, - { - field: 'event.category', - type: 'keyword', - }, - { - field: 'host.name', - type: 'text', - }, - { - field: 'host.name.keyword', - type: 'keyword', - }, - { - field: 'some.field', - type: 'text', - }, - { - field: 'some.field.keyword', - type: 'keyword', - }, - { - field: 'source.ip', - type: 'text', - }, - { - field: 'source.ip.keyword', - type: 'keyword', - }, - { - field: 'source.port', - type: 'long', - }, - ]; - - test('it flattens the field names and types in the mapping properties', () => { - expect(getFieldTypes(mockMappingsProperties)).toEqual(expected); - }); - - test('it throws a type error when mappingsProperties is not flatten-able', () => { - // @ts-expect-error - const invalidType: Record = []; // <-- this is an array, NOT a valid Record - - expect(() => getFieldTypes(invalidType)).toThrowError('Root value is not flatten-able'); - }); - }); - - describe('getIsInSameFamily', () => { - test('it returns false when ecsExpectedType is undefined', () => { - expect(getIsInSameFamily({ ecsExpectedType: undefined, type: 'keyword' })).toBe(false); - }); - - const expectedFamilyMembers: { - [key: string]: string[]; - } = { - constant_keyword: ['keyword', 'wildcard'], // `keyword` and `wildcard` in the same family as `constant_keyword` - keyword: ['constant_keyword', 'wildcard'], - match_only_text: ['text'], - text: ['match_only_text'], - wildcard: ['keyword', 'constant_keyword'], - }; - - const ecsExpectedTypes = Object.keys(expectedFamilyMembers); - - ecsExpectedTypes.forEach((ecsExpectedType) => { - const otherMembersOfSameFamily = expectedFamilyMembers[ecsExpectedType]; - - otherMembersOfSameFamily.forEach((type) => - test(`it returns true for ecsExpectedType '${ecsExpectedType}' when given '${type}', a type in the same family`, () => { - expect(getIsInSameFamily({ ecsExpectedType, type })).toBe(true); - }) - ); - - test(`it returns false for ecsExpectedType '${ecsExpectedType}' when given 'date', a type NOT in the same family`, () => { - expect(getIsInSameFamily({ ecsExpectedType, type: 'date' })).toBe(false); - }); - }); - }); - - describe('isMappingCompatible', () => { - test('it returns true for an exact match', () => { - expect(isMappingCompatible({ ecsExpectedType: 'keyword', type: 'keyword' })).toBe(true); - }); - - test("it returns false when both types don't exactly match", () => { - expect(isMappingCompatible({ ecsExpectedType: 'wildcard', type: 'keyword' })).toBe(false); - }); - }); - - describe('getEnrichedFieldMetadata', () => { - /** - * The ECS schema - * https://raw.githubusercontent.com/elastic/ecs/main/generated/ecs/ecs_flat.yml - * defines a handful of fields that have `allowed_values`. For these - * fields, the documents in an index should only have specific values. - * - * This instance of the type `Record` - * represents an index that doesn't have any unallowed values, for the - * specified keys in the map, i.e. `event.category`, `event.kind`, etc. - * - * This will be used to test the happy path. Variants of this - * value will be used to test unhappy paths. - */ - const noUnallowedValues: Record = { - 'event.category': [], - 'event.kind': [], - 'event.outcome': [], - 'event.type': [], - }; - - /** - * Represents an index that has unallowed values, for the - * `event.category` field. The other fields in the collection, - * i.e. `event.kind`, don't have any unallowed values. - * - * This instance will be used to test paths where a field is - * NOT ECS complaint, because the index has unallowed values. - */ - const unallowedValues: Record = { - 'event.category': [ - { - count: 2, - fieldName: 'an_invalid_category', - }, - { - count: 1, - fieldName: 'theory', - }, - ], - 'event.kind': [], - 'event.outcome': [], - 'event.type': [], - }; - - /** - * This instance of a `FieldType` has the correct mapping for the - * `event.category` field. - * - * This instance will be used to test paths where the index has - * a valid mapping for the `event.category` field. - */ - const fieldMetadataCorrectMappingType: FieldType = { - field: 'event.category', - type: 'keyword', // <-- this index has the correct mapping type - }; - - /** - * This `EnrichedFieldMetadata` for the `event.category` field, - * represents a happy path result, where the index being checked: - * - * 1) The `type` of the field in the index, `keyword`, matches the expected - * `type` of the `event.category` field, as defined by the `EcsMetadata` - * 2) The index doesn't have any unallowed values for the `event.category` field - * - * Since this is a happy path result, it has the following values: - * `indexInvalidValues` is an empty array, because the index does not contain any invalid values - * `isEcsCompliant` is true, because the index has the expected mapping type, and no unallowed values - */ - const happyPathResultSample: EnrichedFieldMetadata = { - dashed_name: 'event-category', - description: - 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', - example: 'authentication', - flat_name: 'event.category', - ignore_above: 1024, - level: 'core', - name: 'category', - normalize: ['array'], - short: 'Event category. The second categorization field in the hierarchy.', - type: 'keyword', - indexFieldName: 'event.category', - indexFieldType: 'keyword', // a valid mapping, because the `type` property from the `ecsMetadata` is also `keyword` - indexInvalidValues: [], // empty array, because the index does not contain any invalid values - hasEcsMetadata: true, - isEcsCompliant: true, // because the index has the expected mapping type, and no unallowed values - isInSameFamily: false, - }; - - /** - * Creates expected result matcher based on the happy path result sample. Please, add similar `expect` based assertions to it if anything breaks - * with an ECS upgrade, instead of hardcoding the values. - */ - const expectedResult = (extraFields: Record = {}) => - expect.objectContaining({ - ...happyPathResultSample, - ...extraFields, - allowed_values: expect.arrayContaining([ - expect.objectContaining({ - description: expect.any(String), - name: expect.any(String), - expected_event_types: expect.any(Array), - }), - ]), - }); - - test('it returns the happy path result when the index has no mapping conflicts, and no unallowed values', () => { - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index - unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index - }) - ).toEqual(expectedResult()); - }); - - test('it returns the happy path result when the index has no mapping conflicts, and the unallowedValues map does not contain an entry for the field', () => { - // create an `unallowedValues` that doesn't have an entry for `event.category`: - const noEntryForEventCategory: Record = omit( - 'event.category', - unallowedValues - ); - - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index - unallowedValues: noEntryForEventCategory, // a lookup in this map for the `event.category` field will return undefined - }) - ).toEqual(expectedResult()); - }); - - test('it returns a result with the expected `indexInvalidValues` and `isEcsCompliant` when the index has no mapping conflict, but it has unallowed values', () => { - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index - unallowedValues, // this index has unallowed values for the event.category field - }) - ).toEqual( - expectedResult({ - indexInvalidValues: [ - { - count: 2, - fieldName: 'an_invalid_category', - }, - { - count: 1, - fieldName: 'theory', - }, - ], - isEcsCompliant: false, // because there are unallowed values - }) - ); - }); - - test('it returns a result with the expected `isEcsCompliant` and `isInSameFamily` when the index type does not match ECS, but NO unallowed values', () => { - const indexFieldType = 'text'; - - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: { - field: 'event.category', // `event.category` is a `keyword`, per the ECS spec - type: indexFieldType, // this index has a mapping of `text` instead - }, - unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index - }) - ).toEqual( - expectedResult({ - indexFieldType, - isEcsCompliant: false, // `keyword` !== `text` - isInSameFamily: false, // `keyword` and `text` are not in the same family - }) - ); - }); - - test('it returns a result with the expected `isEcsCompliant` and `isInSameFamily` when the mapping is is in the same family', () => { - const indexFieldType = 'wildcard'; - - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: { - field: 'event.category', // `event.category` is a `keyword` per the ECS spec - type: indexFieldType, // this index has a mapping of `wildcard` instead - }, - unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index - }) - ).toEqual( - expectedResult({ - indexFieldType, - isEcsCompliant: false, // `wildcard` !== `keyword` - isInSameFamily: true, // `wildcard` and `keyword` are in the same family - }) - ); - }); - - test('it returns a result with the expected `indexInvalidValues`,`isEcsCompliant`, and `isInSameFamily` when the index has BOTH mapping conflicts, and unallowed values', () => { - const indexFieldType = 'text'; - - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: { - field: 'event.category', // `event.category` is a `keyword` per the ECS spec - type: indexFieldType, // this index has a mapping of `text` instead - }, - unallowedValues, // this index also has unallowed values for the event.category field - }) - ).toEqual( - expectedResult({ - indexFieldType, - indexInvalidValues: [ - { - count: 2, - fieldName: 'an_invalid_category', - }, - { - count: 1, - fieldName: 'theory', - }, - ], - isEcsCompliant: false, // because there are BOTH mapping conflicts and unallowed values - isInSameFamily: false, // `text` and `keyword` are not in the same family - }) - ); - }); - - test('it returns the expected result for a custom field, i.e. a field that does NOT have an entry in `ecsMetadata`', () => { - const field = 'a_custom_field'; // not defined by ECS - const indexFieldType = 'keyword'; - - expect( - getEnrichedFieldMetadata({ - ecsMetadata: EcsFlatTyped, - fieldMetadata: { - field, - type: indexFieldType, // no mapping conflict, because ECS doesn't define this field - }, - unallowedValues: noUnallowedValues, // no unallowed values for `a_custom_field` in this index - }) - ).toEqual({ - indexFieldName: field, - indexFieldType, - indexInvalidValues: [], - hasEcsMetadata: false, - isEcsCompliant: false, - isInSameFamily: false, // custom fields are never in the same family - }); - }); - }); - - describe('getMissingTimestampFieldMetadata', () => { - test('it returns the expected `EnrichedFieldMetadata`', () => { - expect(getMissingTimestampFieldMetadata()).toEqual({ - ...EcsFlatTyped['@timestamp'], - hasEcsMetadata: true, - indexFieldName: '@timestamp', - indexFieldType: '-', // the index did NOT define a mapping for @timestamp - indexInvalidValues: [], - isEcsCompliant: false, // an index must define the @timestamp mapping - isInSameFamily: false, // `date` is not a member of any families - }); - }); - }); - - describe('getPartitionedFieldMetadata', () => { - test('it places all the `EnrichedFieldMetadata` in the expected categories', () => { - const enrichedFieldMetadata: EnrichedFieldMetadata[] = [ - timestamp, - eventCategoryWithUnallowedValues, - hostNameWithTextMapping, - hostNameKeyword, - someField, - someFieldKeyword, - sourceIpWithTextMapping, - sourceIpKeyword, - sourcePort, - ]; - const expected: PartitionedFieldMetadata = { - all: [ - timestamp, - eventCategoryWithUnallowedValues, - hostNameWithTextMapping, - hostNameKeyword, - someField, - someFieldKeyword, - sourceIpWithTextMapping, - sourceIpKeyword, - sourcePort, - ], - ecsCompliant: [timestamp, sourcePort], - custom: [hostNameKeyword, someField, someFieldKeyword, sourceIpKeyword], - incompatible: [ - eventCategoryWithUnallowedValues, - hostNameWithTextMapping, - sourceIpWithTextMapping, - ], - sameFamily: [], - }; - - expect(getPartitionedFieldMetadata(enrichedFieldMetadata)).toEqual(expected); - }); - }); - - describe('getPartitionedFieldMetadataStats', () => { - test('it returns the expected stats', () => { - const partitionedFieldMetadata: PartitionedFieldMetadata = { - all: [ - timestamp, - eventCategoryWithUnallowedValues, - hostNameWithTextMapping, - hostNameKeyword, - someField, - someFieldKeyword, - sourceIpWithTextMapping, - sourceIpKeyword, - sourcePort, - ], - ecsCompliant: [timestamp, sourcePort], - custom: [hostNameKeyword, someField, someFieldKeyword, sourceIpKeyword], - incompatible: [ - eventCategoryWithUnallowedValues, - hostNameWithTextMapping, - sourceIpWithTextMapping, - ], - sameFamily: [], - }; - - expect(getPartitionedFieldMetadataStats(partitionedFieldMetadata)).toEqual({ - all: 9, - custom: 4, - ecsCompliant: 2, - incompatible: 3, - sameFamily: 0, - }); - }); - }); - - describe('getDocsCount', () => { - test('it returns the expected docs count when `stats` contains the `indexName`', () => { - const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; - const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs; - - expect( - getDocsCount({ - indexName, - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(expectedCount); - }); - - test('it returns zero when `stats` does NOT contain the `indexName`', () => { - const indexName = 'not-gonna-find-it'; - - expect( - getDocsCount({ - indexName, - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(0); - }); - - test('it returns zero when `stats` is null', () => { - const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; - - expect( - getDocsCount({ - indexName, - stats: null, - }) - ).toEqual(0); - }); - - test('it returns the expected total for a green index, where `primaries.docs.count` and `total.docs.count` have different values', () => { - const indexName = 'auditbeat-custom-index-1'; - - expect( - getDocsCount({ - indexName, - stats: mockStatsAuditbeatIndex, - }) - ).toEqual(mockStatsAuditbeatIndex[indexName].num_docs); - }); - }); - - describe('getSizeInBytes', () => { - test('it returns the expected size when `stats` contains the `indexName`', () => { - const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; - const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes; - - expect( - getSizeInBytes({ - indexName, - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(expectedCount); - }); - - test('it returns undefined when `stats` does NOT contain the `indexName`', () => { - const indexName = 'not-gonna-find-it'; - - expect( - getSizeInBytes({ - indexName, - stats: mockStatsPacketbeatIndex, - }) - ).toBeUndefined(); - }); - - test('it returns undefined when `stats` is null', () => { - const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; - - expect( - getSizeInBytes({ - indexName, - stats: null, - }) - ).toBeUndefined(); - }); - - test('it returns the expected size for a green index, where `primaries.store.size_in_bytes` and `total.store.size_in_bytes` have different values', () => { - const indexName = 'auditbeat-custom-index-1'; - - expect( - getSizeInBytes({ - indexName, - stats: mockStatsAuditbeatIndex, - }) - ).toEqual(mockStatsAuditbeatIndex[indexName].size_in_bytes); - }); - }); - - describe('getTotalDocsCount', () => { - test('it returns the expected total given a subset of index names in the stats', () => { - const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001'; - const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs; - - expect( - getTotalDocsCount({ - indexNames: [indexName], - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(expectedCount); - }); - - test('it returns the expected total given all index names in the stats', () => { - const allIndexNamesInStats = [ - '.ds-packetbeat-8.6.1-2023.02.04-000001', - '.ds-packetbeat-8.5.3-2023.02.04-000001', - ]; - - expect( - getTotalDocsCount({ - indexNames: allIndexNamesInStats, - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(3258632); - }); - - test('it returns zero given an empty collection of index names', () => { - expect( - getTotalDocsCount({ - indexNames: [], // <-- empty - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(0); - }); - - test('it returns the expected total for a green index', () => { - const indexName = 'auditbeat-custom-index-1'; - const expectedCount = mockStatsAuditbeatIndex[indexName].num_docs; - - expect( - getTotalDocsCount({ - indexNames: [indexName], - stats: mockStatsAuditbeatIndex, - }) - ).toEqual(expectedCount); - }); - }); - - describe('getTotalSizeInBytes', () => { - test('it returns the expected total given a subset of index names in the stats', () => { - const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001'; - const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes; - - expect( - getTotalSizeInBytes({ - indexNames: [indexName], - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(expectedCount); - }); - - test('it returns the expected total given all index names in the stats', () => { - const allIndexNamesInStats = [ - '.ds-packetbeat-8.6.1-2023.02.04-000001', - '.ds-packetbeat-8.5.3-2023.02.04-000001', - ]; - - expect( - getTotalSizeInBytes({ - indexNames: allIndexNamesInStats, - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(1464758182); - }); - - test('it returns undefined given an empty collection of index names', () => { - expect( - getTotalSizeInBytes({ - indexNames: [], // <-- empty - stats: mockStatsPacketbeatIndex, - }) - ).toBeUndefined(); - }); - - test('it returns undefined if sizeInByte in not an integer', () => { - const indexName = 'auditbeat-custom-index-1'; - - expect( - getTotalSizeInBytes({ - indexNames: [indexName], - stats: { [indexName]: { ...mockStatsAuditbeatIndex[indexName], size_in_bytes: null } }, - }) - ).toBeUndefined(); - }); - - test('it returns the expected total for an index', () => { - const indexName = 'auditbeat-custom-index-1'; - const expectedCount = mockStatsAuditbeatIndex[indexName].size_in_bytes; - - expect( - getTotalSizeInBytes({ - indexNames: [indexName], - stats: mockStatsAuditbeatIndex, - }) - ).toEqual(expectedCount); - }); - - test('it returns the expected total for indices', () => { - const expectedCount = Object.values(mockStatsPacketbeatIndex).reduce( - (acc, { size_in_bytes: sizeInBytes }) => { - return acc + (sizeInBytes ?? 0); - }, - 0 - ); - - expect( - getTotalSizeInBytes({ - indexNames: [ - '.ds-packetbeat-8.6.1-2023.02.04-000001', - '.ds-packetbeat-8.5.3-2023.02.04-000001', - ], - stats: mockStatsPacketbeatIndex, - }) - ).toEqual(expectedCount); - }); - }); - - describe('getIlmPhaseDescription', () => { - const phases: Array<{ - phase: string; - expected: string; - }> = [ - { - phase: 'hot', - expected: HOT_DESCRIPTION, - }, - { - phase: 'warm', - expected: WARM_DESCRIPTION, - }, - { - phase: 'cold', - expected: COLD_DESCRIPTION, - }, - { - phase: 'frozen', - expected: FROZEN_DESCRIPTION, - }, - { - phase: 'unmanaged', - expected: UNMANAGED_DESCRIPTION, - }, - { - phase: 'something-else', - expected: ' ', - }, - ]; - - phases.forEach(({ phase, expected }) => { - test(`it returns ${expected} when phase is ${phase}`, () => { - expect(getIlmPhaseDescription(phase)).toBe(expected); - }); - }); - }); - - describe('getPatternIlmPhaseDescription', () => { - const phases: Array<{ - expected: string; - indices: number; - pattern: string; - phase: string; - }> = [ - { - expected: - '1 index matching the .alerts-security.alerts-default pattern is hot. Hot indices are actively being updated and queried.', - indices: 1, - pattern: '.alerts-security.alerts-default', - phase: 'hot', - }, - { - expected: - '2 indices matching the .alerts-security.alerts-default pattern are hot. Hot indices are actively being updated and queried.', - indices: 2, - pattern: '.alerts-security.alerts-default', - phase: 'hot', - }, - { - expected: - '1 index matching the .alerts-security.alerts-default pattern is warm. Warm indices are no longer being updated but are still being queried.', - indices: 1, - pattern: '.alerts-security.alerts-default', - phase: 'warm', - }, - { - expected: - '2 indices matching the .alerts-security.alerts-default pattern are warm. Warm indices are no longer being updated but are still being queried.', - indices: 2, - pattern: '.alerts-security.alerts-default', - phase: 'warm', - }, - { - expected: - '1 index matching the .alerts-security.alerts-default pattern is cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.', - indices: 1, - pattern: '.alerts-security.alerts-default', - phase: 'cold', - }, - { - expected: - '2 indices matching the .alerts-security.alerts-default pattern are cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.', - indices: 2, - pattern: '.alerts-security.alerts-default', - phase: 'cold', - }, - { - expected: - "1 index matching the .alerts-security.alerts-default pattern is frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.", - indices: 1, - pattern: '.alerts-security.alerts-default', - phase: 'frozen', - }, - { - expected: - "2 indices matching the .alerts-security.alerts-default pattern are frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.", - indices: 2, - pattern: '.alerts-security.alerts-default', - phase: 'frozen', - }, - { - expected: - '1 index matching the .alerts-security.alerts-default pattern is unmanaged by Index Lifecycle Management (ILM)', - indices: 1, - pattern: '.alerts-security.alerts-default', - phase: 'unmanaged', - }, - { - expected: - '2 indices matching the .alerts-security.alerts-default pattern are unmanaged by Index Lifecycle Management (ILM)', - indices: 2, - pattern: '.alerts-security.alerts-default', - phase: 'unmanaged', - }, - { - expected: '', - indices: 1, - pattern: '.alerts-security.alerts-default', - phase: 'some-other-phase', - }, - { - expected: '', - indices: 2, - pattern: '.alerts-security.alerts-default', - phase: 'some-other-phase', - }, - ]; - - phases.forEach(({ expected, indices, pattern, phase }) => { - test(`it returns the expected description when indices is ${indices}, pattern is ${pattern}, and phase is ${phase}`, () => { - expect(getPatternIlmPhaseDescription({ indices, pattern, phase })).toBe(expected); - }); - }); - }); - - describe('getTotalPatternIncompatible', () => { - test('it returns zero when multiple indices in the results results have a count of zero', () => { - const results: Record = { - '.ds-packetbeat-8.5.3-2023.02.04-000001': { - docsCount: 1630289, - error: null, - ilmPhase: 'hot', - incompatible: 0, - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'packetbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - '.ds-packetbeat-8.6.1-2023.02.04-000001': { - docsCount: 1628343, - error: null, - ilmPhase: 'hot', - incompatible: 0, - indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'packetbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - }; - - expect(getTotalPatternIncompatible(results)).toEqual(0); - }); - - test("it returns the expected total when some indices have incompatible fields, but others don't", () => { - const results: Record = { - '.ds-auditbeat-8.6.1-2023.02.07-000001': { - docsCount: 18086, - error: null, - ilmPhase: 'hot', - incompatible: 0, - indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - 'auditbeat-custom-index-1': { - docsCount: 4, - error: null, - ilmPhase: 'unmanaged', - incompatible: 3, - indexName: 'auditbeat-custom-index-1', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - 'auditbeat-custom-empty-index-1': { - docsCount: 0, - error: null, - ilmPhase: 'unmanaged', - incompatible: 1, - indexName: 'auditbeat-custom-empty-index-1', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - }; - - expect(getTotalPatternIncompatible(results)).toEqual(4); - }); - - test('it returns the expected total when some indices have undefined incompatible counts', () => { - const results: Record = { - '.ds-auditbeat-8.6.1-2023.02.07-000001': { - docsCount: 18086, - error: null, - ilmPhase: 'hot', - incompatible: undefined, // <-- this index has an undefined `incompatible` - indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - 'auditbeat-custom-index-1': { - docsCount: 4, - error: null, - ilmPhase: 'unmanaged', - incompatible: 3, - indexName: 'auditbeat-custom-index-1', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - 'auditbeat-custom-empty-index-1': { - docsCount: 0, - error: null, - ilmPhase: 'unmanaged', - incompatible: 1, - indexName: 'auditbeat-custom-empty-index-1', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'auditbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }, - }; - - expect(getTotalPatternIncompatible(results)).toEqual(4); - }); - - test('it returns zero when `results` is empty', () => { - expect(getTotalPatternIncompatible({})).toEqual(0); - }); - - test('it returns undefined when `results` is undefined', () => { - expect(getTotalPatternIncompatible(undefined)).toBeUndefined(); - }); - }); - - describe('getTotalPatternIndicesChecked', () => { - test('it returns zero when `patternRollup` is undefined', () => { - expect(getTotalPatternIndicesChecked(undefined)).toEqual(0); - }); - - test('it returns zero when `patternRollup` does NOT have any results', () => { - expect(getTotalPatternIndicesChecked(auditbeatNoResults)).toEqual(0); - }); - - test('it returns the expected total when all indices in `patternRollup` have results', () => { - expect(getTotalPatternIndicesChecked(auditbeatWithAllResults)).toEqual(3); - }); - - test('it returns the expected total when some indices in `patternRollup` have errors', () => { - expect(getTotalPatternIndicesChecked(packetbeatWithSomeErrors)).toEqual(1); - }); - }); - - describe('getErrorSummary', () => { - test('it returns the expected error summary', () => { - const resultWithError: DataQualityCheckResult = { - docsCount: 1630289, - error: - 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', - ilmPhase: 'hot', - incompatible: undefined, - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - markdownComments: ['foo', 'bar', 'baz'], - pattern: 'packetbeat-*', - sameFamily: 0, - checkedAt: Date.now(), - }; - - expect(getErrorSummary(resultWithError)).toEqual({ - error: - 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - pattern: 'packetbeat-*', - }); - }); - }); - - describe('getErrorSummariesForRollup', () => { - test('it returns the expected array of `ErrorSummary` when the `PatternRollup` contains errors', () => { - expect(getErrorSummariesForRollup(packetbeatWithSomeErrors)).toEqual([ - { - error: - 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - pattern: 'packetbeat-*', - }, - ]); - }); - - test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` contains all results, with NO errors', () => { - expect(getErrorSummariesForRollup(auditbeatWithAllResults)).toEqual([]); - }); - - test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` has NO results', () => { - expect(getErrorSummariesForRollup(auditbeatNoResults)).toEqual([]); - }); - - test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` is undefined', () => { - expect(getErrorSummariesForRollup(undefined)).toEqual([]); - }); - - test('it returns BOTH the expected (root) pattern-level error, and an index-level error when `PatternRollup` has both', () => { - const withPatternLevelError: PatternRollup = { - ...packetbeatWithSomeErrors, - error: 'This is a pattern-level error', - }; - - expect(getErrorSummariesForRollup(withPatternLevelError)).toEqual([ - { - error: 'This is a pattern-level error', - indexName: null, - pattern: 'packetbeat-*', - }, - { - error: - 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - pattern: 'packetbeat-*', - }, - ]); - }); - - test('it returns the expected (root) pattern-level error when there are no index-level results', () => { - const withPatternLevelError: PatternRollup = { - ...auditbeatNoResults, - error: 'This is a pattern-level error', - }; - - expect(getErrorSummariesForRollup(withPatternLevelError)).toEqual([ - { - error: 'This is a pattern-level error', - indexName: null, - pattern: 'auditbeat-*', - }, - ]); - }); - }); - - describe('getErrorSummaries', () => { - test('it returns an empty array when patternRollups is empty', () => { - expect(getErrorSummaries({})).toEqual([]); - }); - - test('it returns an empty array when none of the patternRollups have errors', () => { - expect( - getErrorSummaries({ - '.alerts-security.alerts-default': alertIndexNoResults, - 'auditbeat-*': auditbeatWithAllResults, - 'packetbeat-*': packetbeatNoResults, - }) - ).toEqual([]); - }); - - test('it returns the expected array of `ErrorSummary` when some of the `PatternRollup` contain errors', () => { - expect( - getErrorSummaries({ - '.alerts-security.alerts-default': alertIndexNoResults, - 'auditbeat-*': auditbeatWithAllResults, - 'packetbeat-*': packetbeatWithSomeErrors, // <-- has errors - }) - ).toEqual([ - { - error: - 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - pattern: 'packetbeat-*', - }, - ]); - }); - - test('it returns the expected array of `ErrorSummary` when there are both pattern-level and index-level errors', () => { - const withPatternLevelError: PatternRollup = { - ...auditbeatNoResults, - error: 'This is a pattern-level error', - }; - - expect( - getErrorSummaries({ - '.alerts-security.alerts-default': alertIndexNoResults, - 'auditbeat-*': withPatternLevelError, // <-- has pattern-level errors - 'packetbeat-*': packetbeatWithSomeErrors, // <-- has index-level errors - }) - ).toEqual([ - { - error: 'This is a pattern-level error', - indexName: null, - pattern: 'auditbeat-*', - }, - { - error: - 'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001', - indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', - pattern: 'packetbeat-*', - }, - ]); - }); - - test('it returns the expected array of `ErrorSummary` when there are just pattern-level errors', () => { - const withPatternLevelError: PatternRollup = { - ...auditbeatNoResults, - error: 'This is a pattern-level error', - }; - - expect( - getErrorSummaries({ - '.alerts-security.alerts-default': alertIndexNoResults, - 'auditbeat-*': withPatternLevelError, // <-- has pattern-level errors - 'packetbeat-*': packetbeatNoResults, - }) - ).toEqual([ - { - error: 'This is a pattern-level error', - indexName: null, - pattern: 'auditbeat-*', - }, - ]); - }); - }); - - describe('formatStorageResult', () => { - it('should correctly format the input data into a StorageResult object', () => { - const inputData: Parameters[number] = { - result: { - indexName: 'testIndex', - pattern: 'testPattern', - checkedAt: 1627545600000, - docsCount: 100, - incompatible: 3, - sameFamily: 1, - ilmPhase: 'hot', - markdownComments: ['test comments'], - error: null, - }, - report: { - batchId: 'testBatch', - isCheckAll: true, - sameFamilyFields: ['agent.type'], - unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], - unallowedValueFields: ['event.category'], - sizeInBytes: 5000, - ecsVersion: '1.0.0', - indexName: 'testIndex', - indexId: 'testIndexId', - }, - partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily, - }; - - const expectedResult: StorageResult = { - batchId: 'testBatch', - indexName: 'testIndex', - indexPattern: 'testPattern', - isCheckAll: true, - checkedAt: 1627545600000, - docsCount: 100, - totalFieldCount: 10, - ecsFieldCount: 2, - customFieldCount: 4, - incompatibleFieldCount: 3, - incompatibleFieldMappingItems: [ - { - fieldName: 'event.category', - expectedValue: 'keyword', - actualValue: 'constant_keyword', - description: - 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', - }, - { - fieldName: 'host.name', - expectedValue: 'keyword', - actualValue: 'text', - description: - 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', - }, - { - fieldName: 'source.ip', - expectedValue: 'ip', - actualValue: 'text', - description: 'IP address of the source (IPv4 or IPv6).', - }, - ], - incompatibleFieldValueItems: [ - { - fieldName: 'event.category', - expectedValues: [ - 'authentication', - 'configuration', - 'database', - 'driver', - 'email', - 'file', - 'host', - 'iam', - 'intrusion_detection', - 'malware', - 'network', - 'package', - 'process', - 'registry', - 'session', - 'threat', - 'vulnerability', - 'web', - ], - actualValues: [{ name: 'an_invalid_category', count: 2 }], - description: - 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', - }, - ], - sameFamilyFieldCount: 1, - sameFamilyFields: ['agent.type'], - sameFamilyFieldItems: [ - { - fieldName: 'agent.type', - expectedValue: 'keyword', - actualValue: 'constant_keyword', - description: - 'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', - }, - ], - unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], - unallowedValueFields: ['event.category'], - sizeInBytes: 5000, - ilmPhase: 'hot', - markdownComments: ['test comments'], - ecsVersion: '1.0.0', - indexId: 'testIndexId', - error: null, - }; - - expect(formatStorageResult(inputData)).toEqual(expectedResult); - }); - }); - - describe('postStorageResult', () => { - const { fetch } = httpServiceMock.createStartContract(); - const { toasts } = notificationServiceMock.createStartContract(); - beforeEach(() => { - fetch.mockClear(); - }); - - test('it posts the result', async () => { - const storageResult = { indexName: 'test' } as unknown as StorageResult; - await postStorageResult({ - storageResult, - httpFetch: fetch, - abortController: new AbortController(), - toasts, - }); - - expect(fetch).toHaveBeenCalledWith( - '/internal/ecs_data_quality_dashboard/results', - expect.objectContaining({ - method: 'POST', - body: JSON.stringify(storageResult), - }) - ); - }); - - test('it throws error', async () => { - const storageResult = { indexName: 'test' } as unknown as StorageResult; - fetch.mockRejectedValueOnce('test-error'); - await postStorageResult({ - httpFetch: fetch, - storageResult, - abortController: new AbortController(), - toasts, - }); - expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); - }); - }); - - describe('getStorageResults', () => { - const { fetch } = httpServiceMock.createStartContract(); - const { toasts } = notificationServiceMock.createStartContract(); - beforeEach(() => { - fetch.mockClear(); - }); - - test('it gets the results', async () => { - await getStorageResults({ - httpFetch: fetch, - abortController: new AbortController(), - pattern: 'auditbeat-*', - toasts, - }); - - expect(fetch).toHaveBeenCalledWith( - '/internal/ecs_data_quality_dashboard/results_latest/auditbeat-*', - expect.objectContaining({ - method: 'GET', - }) - ); - }); - - it('should catch error', async () => { - fetch.mockRejectedValueOnce('test-error'); - - const results = await getStorageResults({ - httpFetch: fetch, - abortController: new AbortController(), - pattern: 'auditbeat-*', - toasts, - }); - - expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); - expect(results).toEqual([]); - }); - }); -}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.ts deleted file mode 100644 index c003da2d5fe70..0000000000000 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.ts +++ /dev/null @@ -1,615 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { HttpHandler } from '@kbn/core-http-browser'; -import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; -import { has, sortBy } from 'lodash/fp'; -import { IToasts } from '@kbn/core-notifications-browser'; -import { getIlmPhase } from './data_quality_details/indices_details/pattern/helpers'; - -import * as i18n from './translations'; - -import type { - DataQualityCheckResult, - DataQualityIndexCheckedParams, - EcsBasedFieldMetadata, - EnrichedFieldMetadata, - ErrorSummary, - IlmPhase, - IncompatibleFieldMappingItem, - IncompatibleFieldValueItem, - MeteringStatsIndex, - PartitionedFieldMetadata, - PartitionedFieldMetadataStats, - PatternRollup, - SameFamilyFieldItem, - UnallowedValueCount, -} from './types'; -import { EcsFlatTyped } from './constants'; - -const EMPTY_INDEX_NAMES: string[] = []; -export const INTERNAL_API_VERSION = '1'; -export const getIndexNames = ({ - ilmExplain, - ilmPhases, - isILMAvailable, - stats, -}: { - ilmExplain: Record | null; - ilmPhases: string[]; - isILMAvailable: boolean; - stats: Record | null; -}): string[] => { - if (((isILMAvailable && ilmExplain != null) || !isILMAvailable) && stats != null) { - const allIndexNames = Object.keys(stats); - const filteredByIlmPhase = isILMAvailable - ? allIndexNames.filter((indexName) => - ilmPhases.includes(getIlmPhase(ilmExplain?.[indexName], isILMAvailable) ?? '') - ) - : allIndexNames; - - return filteredByIlmPhase; - } else { - return EMPTY_INDEX_NAMES; - } -}; - -export interface FieldType { - field: string; - type: string; -} - -function shouldReadKeys(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -const getNextPathWithoutProperties = ({ - key, - pathWithoutProperties, - value, -}: { - key: string; - pathWithoutProperties: string; - value: unknown; -}): string => { - if (!pathWithoutProperties) { - return key; - } - - if (shouldReadKeys(value) && (key === 'properties' || key === 'fields')) { - return `${pathWithoutProperties}`; - } else { - return `${pathWithoutProperties}.${key}`; - } -}; - -export function getFieldTypes(mappingsProperties: Record): FieldType[] { - if (!shouldReadKeys(mappingsProperties)) { - throw new TypeError(`Root value is not flatten-able, received ${mappingsProperties}`); - } - - const result: FieldType[] = []; - (function flatten(prefix, object, pathWithoutProperties) { - for (const [key, value] of Object.entries(object)) { - const path = prefix ? `${prefix}.${key}` : key; - - const nextPathWithoutProperties = getNextPathWithoutProperties({ - key, - pathWithoutProperties, - value, - }); - - if (shouldReadKeys(value)) { - flatten(path, value, nextPathWithoutProperties); - } else { - if (nextPathWithoutProperties.endsWith('.type')) { - const pathWithoutType = nextPathWithoutProperties.slice( - 0, - nextPathWithoutProperties.lastIndexOf('.type') - ); - - result.push({ - field: pathWithoutType, - type: `${value}`, - }); - } - } - } - })('', mappingsProperties, ''); - - return result; -} - -/** - * Per https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html#_core_datatypes - * - * ``` - * Field types are grouped by _family_. Types in the same family have exactly - * the same search behavior but may have different space usage or - * performance characteristics. - * - * Currently, there are two type families, `keyword` and `text`. Other type - * families have only a single field type. For example, the `boolean` type - * family consists of one field type: `boolean`. - * ``` - */ -export const fieldTypeFamilies: Record> = { - keyword: new Set(['keyword', 'constant_keyword', 'wildcard']), - text: new Set(['text', 'match_only_text']), -}; - -export const getIsInSameFamily = ({ - ecsExpectedType, - type, -}: { - ecsExpectedType: string | undefined; - type: string; -}): boolean => { - if (ecsExpectedType != null) { - const allFamilies = Object.values(fieldTypeFamilies); - - return allFamilies.reduce( - (acc, family) => (acc !== true ? family.has(ecsExpectedType) && family.has(type) : acc), - false - ); - } else { - return false; - } -}; - -export const isMappingCompatible = ({ - ecsExpectedType, - type, -}: { - ecsExpectedType: string | undefined; - type: string; -}): boolean => type === ecsExpectedType; - -export const getEnrichedFieldMetadata = ({ - ecsMetadata, - fieldMetadata, - unallowedValues, -}: { - ecsMetadata: EcsFlatTyped; - fieldMetadata: FieldType; - unallowedValues: Record; -}): EnrichedFieldMetadata => { - const { field, type } = fieldMetadata; - const indexInvalidValues = unallowedValues[field] ?? []; - - if (has(fieldMetadata.field, ecsMetadata)) { - const ecsExpectedType = ecsMetadata[field].type; - const isEcsCompliant = - isMappingCompatible({ ecsExpectedType, type }) && indexInvalidValues.length === 0; - - const isInSameFamily = - !isMappingCompatible({ ecsExpectedType, type }) && - indexInvalidValues.length === 0 && - getIsInSameFamily({ ecsExpectedType, type }); - - return { - ...ecsMetadata[field], - indexFieldName: field, - indexFieldType: type, - indexInvalidValues, - hasEcsMetadata: true, - isEcsCompliant, - isInSameFamily, - }; - } else { - return { - indexFieldName: field, - indexFieldType: type, - indexInvalidValues: [], - hasEcsMetadata: false, - isEcsCompliant: false, - isInSameFamily: false, // custom fields are never in the same family - }; - } -}; - -export const getMissingTimestampFieldMetadata = (): EcsBasedFieldMetadata => ({ - ...EcsFlatTyped['@timestamp'], - hasEcsMetadata: true, - indexFieldName: '@timestamp', - indexFieldType: '-', - indexInvalidValues: [], - isEcsCompliant: false, - isInSameFamily: false, // `date` is not a member of any families -}); - -export const getPartitionedFieldMetadata = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): PartitionedFieldMetadata => - enrichedFieldMetadata.reduce( - (acc, x) => ({ - all: [...acc.all, x], - ecsCompliant: x.isEcsCompliant ? [...acc.ecsCompliant, x] : acc.ecsCompliant, - custom: !x.hasEcsMetadata ? [...acc.custom, x] : acc.custom, - incompatible: - x.hasEcsMetadata && !x.isEcsCompliant && !x.isInSameFamily - ? [...acc.incompatible, x] - : acc.incompatible, - sameFamily: x.isInSameFamily ? [...acc.sameFamily, x] : acc.sameFamily, - }), - { - all: [], - ecsCompliant: [], - custom: [], - incompatible: [], - sameFamily: [], - } - ); - -export const getPartitionedFieldMetadataStats = ( - partitionedFieldMetadata: PartitionedFieldMetadata -): PartitionedFieldMetadataStats => { - const { all, ecsCompliant, custom, incompatible, sameFamily } = partitionedFieldMetadata; - - return { - all: all.length, - ecsCompliant: ecsCompliant.length, - custom: custom.length, - incompatible: incompatible.length, - sameFamily: sameFamily.length, - }; -}; - -export const getDocsCount = ({ - indexName, - stats, -}: { - indexName: string; - stats: Record | null; -}): number => (stats && stats[indexName]?.num_docs) ?? 0; - -export const getIndexId = ({ - indexName, - stats, -}: { - indexName: string; - stats: Record | null; -}): string | null | undefined => stats && stats[indexName]?.uuid; - -export const getSizeInBytes = ({ - indexName, - stats, -}: { - indexName: string; - stats: Record | null; -}): number | undefined => (stats && stats[indexName]?.size_in_bytes) ?? undefined; - -export const getTotalDocsCount = ({ - indexNames, - stats, -}: { - indexNames: string[]; - stats: Record | null; -}): number => - indexNames.reduce( - (acc: number, indexName: string) => acc + getDocsCount({ stats, indexName }), - 0 - ); - -export const getTotalSizeInBytes = ({ - indexNames, - stats, -}: { - indexNames: string[]; - stats: Record | null; -}): number | undefined => { - let sum; - for (let i = 0; i < indexNames.length; i++) { - const currentSizeInBytes = getSizeInBytes({ stats, indexName: indexNames[i] }); - if (currentSizeInBytes != null) { - if (sum == null) { - sum = 0; - } - sum += currentSizeInBytes; - } else { - return undefined; - } - } - return sum; -}; - -export const EMPTY_STAT = '--'; - -/** - * Returns an i18n description of an an ILM phase - */ -export const getIlmPhaseDescription = (phase: string): string => { - switch (phase) { - case 'hot': - return i18n.HOT_DESCRIPTION; - case 'warm': - return i18n.WARM_DESCRIPTION; - case 'cold': - return i18n.COLD_DESCRIPTION; - case 'frozen': - return i18n.FROZEN_DESCRIPTION; - case 'unmanaged': - return i18n.UNMANAGED_DESCRIPTION; - default: - return ' '; - } -}; - -export const getPatternIlmPhaseDescription = ({ - indices, - pattern, - phase, -}: { - indices: number; - pattern: string; - phase: string; -}): string => { - switch (phase) { - case 'hot': - return i18n.HOT_PATTERN_TOOLTIP({ indices, pattern }); - case 'warm': - return i18n.WARM_PATTERN_TOOLTIP({ indices, pattern }); - case 'cold': - return i18n.COLD_PATTERN_TOOLTIP({ indices, pattern }); - case 'frozen': - return i18n.FROZEN_PATTERN_TOOLTIP({ indices, pattern }); - case 'unmanaged': - return i18n.UNMANAGED_PATTERN_TOOLTIP({ indices, pattern }); - default: - return ''; - } -}; - -export const getTotalPatternIncompatible = ( - results: Record | undefined -): number | undefined => { - if (results == null) { - return undefined; - } - - const allResults = Object.values(results); - - return allResults.reduce((acc, { incompatible }) => acc + (incompatible ?? 0), 0); -}; - -export const getTotalPatternIndicesChecked = (patternRollup: PatternRollup | undefined): number => { - if (patternRollup != null && patternRollup.results != null) { - const allResults = Object.values(patternRollup.results); - const nonErrorResults = allResults.filter(({ error }) => error == null); - - return nonErrorResults.length; - } else { - return 0; - } -}; - -export const getTotalPatternSameFamily = ( - results: Record | undefined -): number | undefined => { - if (results == null) { - return undefined; - } - - const allResults = Object.values(results); - - return allResults.reduce((acc, { sameFamily }) => acc + (sameFamily ?? 0), 0); -}; - -export const getIncompatibleStatBadgeColor = (incompatible: number | undefined): string => - incompatible != null && incompatible > 0 ? 'danger' : 'hollow'; - -export const getErrorSummary = ({ - error, - indexName, - pattern, -}: DataQualityCheckResult): ErrorSummary => ({ - error: String(error), - indexName, - pattern, -}); - -export const getErrorSummariesForRollup = ( - patternRollup: PatternRollup | undefined -): ErrorSummary[] => { - const maybePatternErrorSummary: ErrorSummary[] = - patternRollup != null && patternRollup.error != null - ? [{ pattern: patternRollup.pattern, indexName: null, error: patternRollup.error }] - : []; - - if (patternRollup != null && patternRollup.results != null) { - const unsortedResults: DataQualityCheckResult[] = Object.values(patternRollup.results); - const sortedResults = sortBy('indexName', unsortedResults); - - return sortedResults.reduce( - (acc, result) => [...acc, ...(result.error != null ? [getErrorSummary(result)] : [])], - maybePatternErrorSummary - ); - } else { - return maybePatternErrorSummary; - } -}; - -export const getErrorSummaries = ( - patternRollups: Record -): ErrorSummary[] => { - const allPatterns: string[] = Object.keys(patternRollups); - - // sort the patterns A-Z: - const sortedPatterns = [...allPatterns].sort((a, b) => { - return a.localeCompare(b); - }); - - return sortedPatterns.reduce( - (acc, pattern) => [...acc, ...getErrorSummariesForRollup(patternRollups[pattern])], - [] - ); -}; - -export const POST_INDEX_RESULTS = '/internal/ecs_data_quality_dashboard/results'; -export const GET_INDEX_RESULTS_LATEST = - '/internal/ecs_data_quality_dashboard/results_latest/{pattern}'; - -export interface StorageResult { - batchId: string; - indexName: string; - indexPattern: string; - isCheckAll: boolean; - checkedAt: number; - docsCount: number; - totalFieldCount: number; - ecsFieldCount: number; - customFieldCount: number; - incompatibleFieldCount: number; - incompatibleFieldMappingItems: IncompatibleFieldMappingItem[]; - incompatibleFieldValueItems: IncompatibleFieldValueItem[]; - sameFamilyFieldCount: number; - sameFamilyFields: string[]; - sameFamilyFieldItems: SameFamilyFieldItem[]; - unallowedMappingFields: string[]; - unallowedValueFields: string[]; - sizeInBytes: number; - ilmPhase?: IlmPhase; - markdownComments: string[]; - ecsVersion: string; - indexId: string; - error: string | null; -} - -export const formatStorageResult = ({ - result, - report, - partitionedFieldMetadata, -}: { - result: DataQualityCheckResult; - report: DataQualityIndexCheckedParams; - partitionedFieldMetadata: PartitionedFieldMetadata; -}): StorageResult => { - const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = []; - const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = []; - const sameFamilyFieldItems: SameFamilyFieldItem[] = []; - - partitionedFieldMetadata.incompatible.forEach((field) => { - if (field.type !== field.indexFieldType) { - incompatibleFieldMappingItems.push({ - fieldName: field.indexFieldName, - expectedValue: field.type, - actualValue: field.indexFieldType, - description: field.description, - }); - } - - if (field.indexInvalidValues.length > 0) { - incompatibleFieldValueItems.push({ - fieldName: field.indexFieldName, - expectedValues: field.allowed_values?.map((x) => x.name) ?? [], - actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })), - description: field.description, - }); - } - }); - - partitionedFieldMetadata.sameFamily.forEach((field) => { - sameFamilyFieldItems.push({ - fieldName: field.indexFieldName, - expectedValue: field.type, - actualValue: field.indexFieldType, - description: field.description, - }); - }); - - return { - batchId: report.batchId, - indexName: result.indexName, - indexPattern: result.pattern, - isCheckAll: report.isCheckAll, - checkedAt: result.checkedAt ?? Date.now(), - docsCount: result.docsCount ?? 0, - totalFieldCount: partitionedFieldMetadata.all.length, - ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, - customFieldCount: partitionedFieldMetadata.custom.length, - incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, - incompatibleFieldMappingItems, - incompatibleFieldValueItems, - sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, - sameFamilyFields: report.sameFamilyFields ?? [], - sameFamilyFieldItems, - unallowedMappingFields: report.unallowedMappingFields ?? [], - unallowedValueFields: report.unallowedValueFields ?? [], - sizeInBytes: report.sizeInBytes ?? 0, - ilmPhase: result.ilmPhase, - markdownComments: result.markdownComments, - ecsVersion: report.ecsVersion, - indexId: report.indexId ?? '', - error: result.error, - }; -}; - -export const formatResultFromStorage = ({ - storageResult, - pattern, -}: { - storageResult: StorageResult; - pattern: string; -}): DataQualityCheckResult => ({ - docsCount: storageResult.docsCount, - error: storageResult.error, - ilmPhase: storageResult.ilmPhase, - incompatible: storageResult.incompatibleFieldCount, - indexName: storageResult.indexName, - markdownComments: storageResult.markdownComments, - sameFamily: storageResult.sameFamilyFieldCount, - checkedAt: storageResult.checkedAt, - pattern, -}); - -export async function postStorageResult({ - storageResult, - httpFetch, - toasts, - abortController = new AbortController(), -}: { - storageResult: StorageResult; - httpFetch: HttpHandler; - toasts: IToasts; - abortController?: AbortController; -}): Promise { - try { - await httpFetch(POST_INDEX_RESULTS, { - method: 'POST', - signal: abortController.signal, - version: INTERNAL_API_VERSION, - body: JSON.stringify(storageResult), - }); - } catch (err) { - toasts.addError(err, { title: i18n.POST_RESULT_ERROR_TITLE }); - } -} - -export async function getStorageResults({ - pattern, - httpFetch, - toasts, - abortController, -}: { - pattern: string; - httpFetch: HttpHandler; - toasts: IToasts; - abortController: AbortController; -}): Promise { - try { - const route = GET_INDEX_RESULTS_LATEST.replace('{pattern}', pattern); - const results = await httpFetch(route, { - method: 'GET', - signal: abortController.signal, - version: INTERNAL_API_VERSION, - }); - return results; - } catch (err) { - toasts.addError(err, { title: i18n.GET_RESULTS_ERROR_TITLE }); - return []; - } -} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/constants.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/constants.ts new file mode 100644 index 0000000000000..e2251d526136f --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/constants.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const POST_INDEX_RESULTS = '/internal/ecs_data_quality_dashboard/results'; +export const GET_INDEX_RESULTS_LATEST = + '/internal/ecs_data_quality_dashboard/results_latest/{pattern}'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx index 0828d496f700d..eb8cd670d70c5 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx @@ -7,10 +7,10 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { EcsVersion } from '@elastic/ecs'; - import { isEmpty } from 'lodash/fp'; import { IToasts } from '@kbn/core-notifications-browser'; import { HttpHandler } from '@kbn/core-http-browser'; + import { getTotalDocsCount, getTotalIncompatible, @@ -18,25 +18,22 @@ import { getTotalIndicesChecked, getTotalSameFamily, getTotalSizeInBytes, - updateResultOnCheckCompleted, -} from './helpers'; - + getTotalPatternSameFamily, + getIndexId, +} from './utils/stats'; +import { + getStorageResults, + postStorageResult, + formatStorageResult, + formatResultFromStorage, +} from './utils/storage'; +import { getPatternRollupsWithLatestCheckResult } from './utils/get_pattern_rollups_with_latest_check_result'; import type { DataQualityCheckResult, OnCheckCompleted, PatternRollup, TelemetryEvents, } from '../../types'; -import { - getDocsCount, - getIndexId, - getStorageResults, - getSizeInBytes, - getTotalPatternSameFamily, - postStorageResult, - formatStorageResult, - formatResultFromStorage, -} from '../../helpers'; import { getIlmPhase, getIndexIncompatible, @@ -48,6 +45,7 @@ import { } from '../../data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers'; import { UseResultsRollupReturnValue } from './types'; import { useIsMounted } from '../use_is_mounted'; +import { getDocsCount, getSizeInBytes } from '../../utils/stats'; interface Props { ilmPhases: string[]; @@ -172,7 +170,7 @@ export const useResultsRollup = ({ isCheckAll, }) => { setPatternRollups((currentPatternRollups) => { - const updatedRollups = updateResultOnCheckCompleted({ + const updatedRollups = getPatternRollupsWithLatestCheckResult({ error, formatBytes, formatNumber, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/types.ts index a1e9a27418a89..093ee8fdd645c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/types.ts @@ -5,7 +5,14 @@ * 2.0. */ -import { OnCheckCompleted, PatternRollup } from '../../types'; +import { + IlmPhase, + IncompatibleFieldMappingItem, + IncompatibleFieldValueItem, + OnCheckCompleted, + PatternRollup, + SameFamilyFieldItem, +} from '../../types'; export interface UseResultsRollupReturnValue { onCheckCompleted: OnCheckCompleted; @@ -26,3 +33,29 @@ export interface UseResultsRollupReturnValue { }) => void; updatePatternRollup: (patternRollup: PatternRollup) => void; } + +export interface StorageResult { + batchId: string; + indexName: string; + indexPattern: string; + isCheckAll: boolean; + checkedAt: number; + docsCount: number; + totalFieldCount: number; + ecsFieldCount: number; + customFieldCount: number; + incompatibleFieldCount: number; + incompatibleFieldMappingItems: IncompatibleFieldMappingItem[]; + incompatibleFieldValueItems: IncompatibleFieldValueItem[]; + sameFamilyFieldCount: number; + sameFamilyFields: string[]; + sameFamilyFieldItems: SameFamilyFieldItem[]; + unallowedMappingFields: string[]; + unallowedValueFields: string[]; + sizeInBytes: number; + ilmPhase?: IlmPhase; + markdownComments: string[]; + ecsVersion: string; + indexId: string; + error: string | null; +} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.test.ts similarity index 77% rename from x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.test.ts rename to x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.test.ts index 2185ec35cddbb..f6cd702ef139f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.test.ts @@ -7,24 +7,11 @@ import numeral from '@elastic/numeral'; -import { - getTotalDocsCount, - getTotalIncompatible, - getTotalIndices, - getTotalIndicesChecked, - getTotalSameFamily, - updateResultOnCheckCompleted, -} from './helpers'; -import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; -import { - mockPacketbeatPatternRollup, - packetbeatNoResults, - packetbeatWithSomeErrors, -} from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; -import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../../types'; -import { EMPTY_STAT } from '../../helpers'; -import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; -import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup'; +import { getPatternRollupsWithLatestCheckResult } from './get_pattern_rollups_with_latest_check_result'; +import { mockPacketbeatPatternRollup } from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; +import { MeteringStatsIndex, PatternRollup } from '../../../types'; +import { EMPTY_STAT } from '../../../constants'; +import { mockPartitionedFieldMetadata } from '../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; import { EcsVersion } from '@elastic/ecs'; const defaultBytesFormat = '0,0.[0]b'; @@ -35,11 +22,6 @@ const defaultNumberFormat = '0,0.[000]'; const formatNumber = (value: number | undefined) => value != null ? numeral(value).format(defaultNumberFormat) : EMPTY_STAT; -const patternRollups: Record = { - 'auditbeat-*': auditbeatWithAllResults, // indices: 3 - 'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2 -}; - describe('helpers', () => { let originalFetch: (typeof global)['fetch']; @@ -51,121 +33,6 @@ describe('helpers', () => { global.fetch = originalFetch; }); - describe('getTotalSameFamily', () => { - const defaultDataQualityCheckResult: DataQualityCheckResult = { - docsCount: 26093, - error: null, - ilmPhase: 'hot', - incompatible: 0, - indexName: '.internal.alerts-security.alerts-default-000001', - markdownComments: ['foo', 'bar', 'baz'], - pattern: '.alerts-security.alerts-default', - sameFamily: 7, - checkedAt: 1706526408000, - }; - - const alertIndexWithSameFamily: PatternRollup = { - ...alertIndexWithAllResults, - results: { - '.internal.alerts-security.alerts-default-000001': { - ...defaultDataQualityCheckResult, - }, - }, - }; - - const withSameFamily: Record = { - '.internal.alerts-security.alerts-default-000001': alertIndexWithSameFamily, - }; - - test('it returns the expected count when patternRollups has sameFamily', () => { - expect(getTotalSameFamily(withSameFamily)).toEqual(7); - }); - - test('it returns undefined when patternRollups is empty', () => { - expect(getTotalSameFamily({})).toBeUndefined(); - }); - - test('it returns zero when none of the rollups have same family', () => { - expect(getTotalSameFamily(patternRollups)).toEqual(0); - }); - }); - - describe('getTotalIndices', () => { - test('it returns the expected total when ALL `PatternRollup`s have an `indices`', () => { - expect(getTotalIndices(patternRollups)).toEqual(5); - }); - - test('it returns undefined when only SOME of the `PatternRollup`s have an `indices`', () => { - const someIndicesAreUndefined: Record = { - 'auditbeat-*': { - ...auditbeatWithAllResults, - indices: undefined, // <-- - }, - 'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2 - }; - - expect(getTotalIndices(someIndicesAreUndefined)).toBeUndefined(); - }); - }); - - describe('getTotalDocsCount', () => { - test('it returns the expected total when ALL `PatternRollup`s have a `docsCount`', () => { - expect(getTotalDocsCount(patternRollups)).toEqual( - Number(auditbeatWithAllResults.docsCount) + Number(mockPacketbeatPatternRollup.docsCount) - ); - }); - - test('it returns undefined when only SOME of the `PatternRollup`s have a `docsCount`', () => { - const someIndicesAreUndefined: Record = { - 'auditbeat-*': { - ...auditbeatWithAllResults, - docsCount: undefined, // <-- - }, - 'packetbeat-*': mockPacketbeatPatternRollup, - }; - - expect(getTotalDocsCount(someIndicesAreUndefined)).toBeUndefined(); - }); - }); - - describe('getTotalIncompatible', () => { - test('it returns the expected total when ALL `PatternRollup`s have `results`', () => { - expect(getTotalIncompatible(patternRollups)).toEqual(4); - }); - - test('it returns the expected total when only SOME of the `PatternRollup`s have `results`', () => { - const someResultsAreUndefined: Record = { - 'auditbeat-*': auditbeatWithAllResults, - 'packetbeat-*': packetbeatNoResults, // <-- results is undefined - }; - - expect(getTotalIncompatible(someResultsAreUndefined)).toEqual(4); - }); - - test('it returns undefined when NONE of the `PatternRollup`s have `results`', () => { - const someResultsAreUndefined: Record = { - 'packetbeat-*': packetbeatNoResults, // <-- results is undefined - }; - - expect(getTotalIncompatible(someResultsAreUndefined)).toBeUndefined(); - }); - }); - - describe('getTotalIndicesChecked', () => { - test('it returns the expected total', () => { - expect(getTotalIndicesChecked(patternRollups)).toEqual(3); - }); - - test('it returns the expected total when errors have occurred', () => { - const someErrors: Record = { - 'auditbeat-*': auditbeatWithAllResults, // indices: 3 - 'packetbeat-*': packetbeatWithSomeErrors, // <-- indices: 2, but one has errors - }; - - expect(getTotalIndicesChecked(someErrors)).toEqual(4); - }); - }); - describe('updateResultOnCheckCompleted', () => { const packetbeatStats861: MeteringStatsIndex = mockPacketbeatPatternRollup.stats != null @@ -178,7 +45,7 @@ describe('helpers', () => { test('it returns the updated rollups', () => { expect( - updateResultOnCheckCompleted({ + getPatternRollupsWithLatestCheckResult({ error: null, formatBytes, formatNumber, @@ -280,7 +147,7 @@ describe('helpers', () => { }; expect( - updateResultOnCheckCompleted({ + getPatternRollupsWithLatestCheckResult({ error: null, formatBytes, formatNumber, @@ -377,7 +244,7 @@ describe('helpers', () => { test('it returns the expected results when `partitionedFieldMetadata` is null', () => { expect( - updateResultOnCheckCompleted({ + getPatternRollupsWithLatestCheckResult({ error: null, formatBytes, formatNumber, @@ -472,7 +339,7 @@ describe('helpers', () => { }; expect( - updateResultOnCheckCompleted({ + getPatternRollupsWithLatestCheckResult({ error: null, formatBytes, formatNumber, @@ -532,7 +399,7 @@ describe('helpers', () => { }; expect( - updateResultOnCheckCompleted({ + getPatternRollupsWithLatestCheckResult({ error: null, formatBytes, formatNumber, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.ts new file mode 100644 index 0000000000000..22b7eb6db4d8d --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getIndexDocsCountFromRollup } from '../../../data_quality_summary/summary_actions/check_all/helpers'; +import { getIlmPhase } from '../../../data_quality_details/indices_details/pattern/helpers'; +import { getAllIncompatibleMarkdownComments } from '../../../data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers'; +import { getSizeInBytes } from '../../../utils/stats'; +import type { IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../../types'; + +export const getPatternRollupsWithLatestCheckResult = ({ + error, + formatBytes, + formatNumber, + indexName, + isILMAvailable, + partitionedFieldMetadata, + pattern, + patternRollups, +}: { + error: string | null; + formatBytes: (value: number | undefined) => string; + formatNumber: (value: number | undefined) => string; + indexName: string; + isILMAvailable: boolean; + partitionedFieldMetadata: PartitionedFieldMetadata | null; + pattern: string; + patternRollups: Record; +}): Record => { + const patternRollup: PatternRollup | undefined = patternRollups[pattern]; + + if (patternRollup != null) { + const ilmExplain = patternRollup.ilmExplain; + + const ilmPhase: IlmPhase | undefined = + ilmExplain != null ? getIlmPhase(ilmExplain[indexName], isILMAvailable) : undefined; + + const docsCount = getIndexDocsCountFromRollup({ + indexName, + patternRollup, + }); + + const patternDocsCount = patternRollup.docsCount ?? 0; + + const sizeInBytes = getSizeInBytes({ indexName, stats: patternRollup.stats }); + + const markdownComments = + partitionedFieldMetadata != null + ? getAllIncompatibleMarkdownComments({ + docsCount, + formatBytes, + formatNumber, + ilmPhase, + indexName, + isILMAvailable, + partitionedFieldMetadata, + patternDocsCount, + sizeInBytes, + }) + : []; + + const incompatible = partitionedFieldMetadata?.incompatible.length; + const sameFamily = partitionedFieldMetadata?.sameFamily.length; + const checkedAt = partitionedFieldMetadata ? Date.now() : undefined; + + return { + ...patternRollups, + [pattern]: { + ...patternRollup, + results: { + ...(patternRollup.results ?? {}), + [indexName]: { + docsCount, + error, + ilmPhase, + incompatible, + indexName, + markdownComments, + pattern, + sameFamily, + checkedAt, + }, + }, + }, + }; + } else { + return patternRollups; + } +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.test.ts new file mode 100644 index 0000000000000..7f28c6bcd1727 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.test.ts @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertIndexWithAllResults } from '../../../mock/pattern_rollup/mock_alerts_pattern_rollup'; +import { auditbeatWithAllResults } from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; +import { + mockPacketbeatPatternRollup, + packetbeatNoResults, + packetbeatWithSomeErrors, +} from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; +import { mockStats } from '../../../mock/stats/mock_stats'; +import { DataQualityCheckResult, PatternRollup } from '../../../types'; +import { + getIndexId, + getTotalDocsCount, + getTotalIncompatible, + getTotalIndices, + getTotalIndicesChecked, + getTotalPatternSameFamily, + getTotalSameFamily, +} from './stats'; + +const patternRollups: Record = { + 'auditbeat-*': auditbeatWithAllResults, // indices: 3 + 'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2 +}; + +describe('getTotalPatternSameFamily', () => { + const baseResult: DataQualityCheckResult = { + docsCount: 4, + error: null, + ilmPhase: 'unmanaged', + incompatible: 3, + indexName: 'auditbeat-custom-index-1', + markdownComments: [ + '### auditbeat-custom-index-1\n', + '| Result | Index | Docs | Incompatible fields | ILM Phase |\n|--------|-------|------|---------------------|-----------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | `unmanaged` |\n\n', + '### **Incompatible fields** `3` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n', + "#### 3 incompatible fields, 0 fields with mappings in the same family\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\nIncompatible fields with mappings in the same family have exactly the same search behavior but may have different space usage or performance characteristics.\n\nWhen an incompatible field is not in the same family:\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", + '\n#### Incompatible field mappings - auditbeat-custom-index-1\n\n\n| Field | ECS mapping type (expected) | Index mapping type (actual) | \n|-------|-----------------------------|-----------------------------|\n| host.name | `keyword` | `text` |\n| source.ip | `ip` | `text` |\n\n#### Incompatible field values - auditbeat-custom-index-1\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `an_invalid_category` (2),\n`theory` (1) |\n\n', + ], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }; + + it('returns undefined when results is undefined', () => { + expect(getTotalPatternSameFamily(undefined)).toBeUndefined(); + }); + + it('returns 0 when results is an empty object', () => { + expect(getTotalPatternSameFamily({})).toBe(0); + }); + + it('should sum sameFamily values and return the total', () => { + const results: Record = { + a: { + ...baseResult, + indexName: 'a', + markdownComments: [], + pattern: 'pattern', + sameFamily: 2, + }, + b: { + ...baseResult, + indexName: 'b', + markdownComments: [], + pattern: 'pattern', + sameFamily: 3, + }, + c: { ...baseResult, indexName: 'c', markdownComments: [], pattern: 'pattern' }, + }; + + expect(getTotalPatternSameFamily(results)).toBe(5); + }); + + it('handles a mix of defined and undefined sameFamily values', () => { + const results: Record = { + a: { + ...baseResult, + indexName: 'a', + markdownComments: [], + pattern: 'pattern', + sameFamily: 1, + }, + b: { + ...baseResult, + indexName: 'b', + markdownComments: [], + pattern: 'pattern', + sameFamily: undefined, + }, + c: { + ...baseResult, + indexName: 'c', + markdownComments: [], + pattern: 'pattern', + sameFamily: 2, + }, + }; + + expect(getTotalPatternSameFamily(results)).toBe(3); + }); +}); + +describe('getTotalSameFamily', () => { + const defaultDataQualityCheckResult: DataQualityCheckResult = { + docsCount: 26093, + error: null, + ilmPhase: 'hot', + incompatible: 0, + indexName: '.internal.alerts-security.alerts-default-000001', + markdownComments: ['foo', 'bar', 'baz'], + pattern: '.alerts-security.alerts-default', + sameFamily: 7, + checkedAt: 1706526408000, + }; + + const alertIndexWithSameFamily: PatternRollup = { + ...alertIndexWithAllResults, + results: { + '.internal.alerts-security.alerts-default-000001': { + ...defaultDataQualityCheckResult, + }, + }, + }; + + const withSameFamily: Record = { + '.internal.alerts-security.alerts-default-000001': alertIndexWithSameFamily, + }; + + test('it returns the expected count when patternRollups has sameFamily', () => { + expect(getTotalSameFamily(withSameFamily)).toEqual(7); + }); + + test('it returns undefined when patternRollups is empty', () => { + expect(getTotalSameFamily({})).toBeUndefined(); + }); + + test('it returns zero when none of the rollups have same family', () => { + expect(getTotalSameFamily(patternRollups)).toEqual(0); + }); +}); + +describe('getTotalIndices', () => { + test('it returns the expected total when ALL `PatternRollup`s have an `indices`', () => { + expect(getTotalIndices(patternRollups)).toEqual(5); + }); + + test('it returns undefined when only SOME of the `PatternRollup`s have an `indices`', () => { + const someIndicesAreUndefined: Record = { + 'auditbeat-*': { + ...auditbeatWithAllResults, + indices: undefined, // <-- + }, + 'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2 + }; + + expect(getTotalIndices(someIndicesAreUndefined)).toBeUndefined(); + }); +}); + +describe('getTotalDocsCount', () => { + test('it returns the expected total when ALL `PatternRollup`s have a `docsCount`', () => { + expect(getTotalDocsCount(patternRollups)).toEqual( + Number(auditbeatWithAllResults.docsCount) + Number(mockPacketbeatPatternRollup.docsCount) + ); + }); + + test('it returns undefined when only SOME of the `PatternRollup`s have a `docsCount`', () => { + const someIndicesAreUndefined: Record = { + 'auditbeat-*': { + ...auditbeatWithAllResults, + docsCount: undefined, // <-- + }, + 'packetbeat-*': mockPacketbeatPatternRollup, + }; + + expect(getTotalDocsCount(someIndicesAreUndefined)).toBeUndefined(); + }); +}); + +describe('getTotalIncompatible', () => { + test('it returns the expected total when ALL `PatternRollup`s have `results`', () => { + expect(getTotalIncompatible(patternRollups)).toEqual(4); + }); + + test('it returns the expected total when only SOME of the `PatternRollup`s have `results`', () => { + const someResultsAreUndefined: Record = { + 'auditbeat-*': auditbeatWithAllResults, + 'packetbeat-*': packetbeatNoResults, // <-- results is undefined + }; + + expect(getTotalIncompatible(someResultsAreUndefined)).toEqual(4); + }); + + test('it returns undefined when NONE of the `PatternRollup`s have `results`', () => { + const someResultsAreUndefined: Record = { + 'packetbeat-*': packetbeatNoResults, // <-- results is undefined + }; + + expect(getTotalIncompatible(someResultsAreUndefined)).toBeUndefined(); + }); +}); + +describe('getTotalIndicesChecked', () => { + test('it returns the expected total', () => { + expect(getTotalIndicesChecked(patternRollups)).toEqual(3); + }); + + test('it returns the expected total when errors have occurred', () => { + const someErrors: Record = { + 'auditbeat-*': auditbeatWithAllResults, // indices: 3 + 'packetbeat-*': packetbeatWithSomeErrors, // <-- indices: 2, but one has errors + }; + + expect(getTotalIndicesChecked(someErrors)).toEqual(4); + }); +}); + +describe('getIndexId', () => { + it('returns the expected index ID', () => { + expect(getIndexId({ indexName: 'auditbeat-custom-index-1', stats: mockStats })).toEqual( + 'uyJDDqGrRQqdBTN0mCF-iw' + ); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.ts similarity index 52% rename from x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.ts rename to x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.ts index 4363633b9f530..c3a44f04471ea 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.ts @@ -5,16 +5,20 @@ * 2.0. */ -import { getIndexDocsCountFromRollup } from '../../data_quality_summary/summary_actions/check_all/helpers'; -import { getIlmPhase } from '../../data_quality_details/indices_details/pattern/helpers'; -import { getAllIncompatibleMarkdownComments } from '../../data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers'; -import { - getSizeInBytes, - getTotalPatternIncompatible, - getTotalPatternIndicesChecked, - getTotalPatternSameFamily, -} from '../../helpers'; -import type { IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../types'; +import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../../../types'; +import { getTotalPatternIncompatible, getTotalPatternIndicesChecked } from '../../../utils/stats'; + +export const getTotalPatternSameFamily = ( + results: Record | undefined +): number | undefined => { + if (results == null) { + return undefined; + } + + const allResults = Object.values(results); + + return allResults.reduce((acc, { sameFamily }) => acc + (sameFamily ?? 0), 0); +}; export const getTotalIndices = ( patternRollups: Record @@ -87,82 +91,10 @@ export const getTotalIndicesChecked = (patternRollups: Record string; - formatNumber: (value: number | undefined) => string; indexName: string; - isILMAvailable: boolean; - partitionedFieldMetadata: PartitionedFieldMetadata | null; - pattern: string; - patternRollups: Record; -}): Record => { - const patternRollup: PatternRollup | undefined = patternRollups[pattern]; - - if (patternRollup != null) { - const ilmExplain = patternRollup.ilmExplain; - - const ilmPhase: IlmPhase | undefined = - ilmExplain != null ? getIlmPhase(ilmExplain[indexName], isILMAvailable) : undefined; - - const docsCount = getIndexDocsCountFromRollup({ - indexName, - patternRollup, - }); - - const patternDocsCount = patternRollup.docsCount ?? 0; - - const sizeInBytes = getSizeInBytes({ indexName, stats: patternRollup.stats }); - - const markdownComments = - partitionedFieldMetadata != null - ? getAllIncompatibleMarkdownComments({ - docsCount, - formatBytes, - formatNumber, - ilmPhase, - indexName, - isILMAvailable, - partitionedFieldMetadata, - patternDocsCount, - sizeInBytes, - }) - : []; - - const incompatible = partitionedFieldMetadata?.incompatible.length; - const sameFamily = partitionedFieldMetadata?.sameFamily.length; - const checkedAt = partitionedFieldMetadata ? Date.now() : undefined; - - return { - ...patternRollups, - [pattern]: { - ...patternRollup, - results: { - ...(patternRollup.results ?? {}), - [indexName]: { - docsCount, - error, - ilmPhase, - incompatible, - indexName, - markdownComments, - pattern, - sameFamily, - checkedAt, - }, - }, - }, - }; - } else { - return patternRollups; - } -}; + stats: Record | null; +}): string | null | undefined => stats && stats[indexName]?.uuid; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts new file mode 100644 index 0000000000000..0e894694ed1e9 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServiceMock } from '@kbn/core-http-browser-mocks'; +import { mockPartitionedFieldMetadataWithSameFamily } from '../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family'; +import { StorageResult } from '../types'; +import { formatStorageResult, getStorageResults, postStorageResult } from './storage'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; + +describe('formatStorageResult', () => { + it('should correctly format the input data into a StorageResult object', () => { + const inputData: Parameters[number] = { + result: { + indexName: 'testIndex', + pattern: 'testPattern', + checkedAt: 1627545600000, + docsCount: 100, + incompatible: 3, + sameFamily: 1, + ilmPhase: 'hot', + markdownComments: ['test comments'], + error: null, + }, + report: { + batchId: 'testBatch', + isCheckAll: true, + sameFamilyFields: ['agent.type'], + unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], + unallowedValueFields: ['event.category'], + sizeInBytes: 5000, + ecsVersion: '1.0.0', + indexName: 'testIndex', + indexId: 'testIndexId', + }, + partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily, + }; + + const expectedResult: StorageResult = { + batchId: 'testBatch', + indexName: 'testIndex', + indexPattern: 'testPattern', + isCheckAll: true, + checkedAt: 1627545600000, + docsCount: 100, + totalFieldCount: 10, + ecsFieldCount: 2, + customFieldCount: 4, + incompatibleFieldCount: 3, + incompatibleFieldMappingItems: [ + { + fieldName: 'event.category', + expectedValue: 'keyword', + actualValue: 'constant_keyword', + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + }, + { + fieldName: 'host.name', + expectedValue: 'keyword', + actualValue: 'text', + description: + 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + }, + { + fieldName: 'source.ip', + expectedValue: 'ip', + actualValue: 'text', + description: 'IP address of the source (IPv4 or IPv6).', + }, + ], + incompatibleFieldValueItems: [ + { + fieldName: 'event.category', + expectedValues: [ + 'authentication', + 'configuration', + 'database', + 'driver', + 'email', + 'file', + 'host', + 'iam', + 'intrusion_detection', + 'malware', + 'network', + 'package', + 'process', + 'registry', + 'session', + 'threat', + 'vulnerability', + 'web', + ], + actualValues: [{ name: 'an_invalid_category', count: 2 }], + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + }, + ], + sameFamilyFieldCount: 1, + sameFamilyFields: ['agent.type'], + sameFamilyFieldItems: [ + { + fieldName: 'agent.type', + expectedValue: 'keyword', + actualValue: 'constant_keyword', + description: + 'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + }, + ], + unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], + unallowedValueFields: ['event.category'], + sizeInBytes: 5000, + ilmPhase: 'hot', + markdownComments: ['test comments'], + ecsVersion: '1.0.0', + indexId: 'testIndexId', + error: null, + }; + + expect(formatStorageResult(inputData)).toEqual(expectedResult); + }); +}); + +describe('postStorageResult', () => { + const { fetch } = httpServiceMock.createStartContract(); + const { toasts } = notificationServiceMock.createStartContract(); + beforeEach(() => { + fetch.mockClear(); + }); + + test('it posts the result', async () => { + const storageResult = { indexName: 'test' } as unknown as StorageResult; + await postStorageResult({ + storageResult, + httpFetch: fetch, + abortController: new AbortController(), + toasts, + }); + + expect(fetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results', + expect.objectContaining({ + method: 'POST', + body: JSON.stringify(storageResult), + }) + ); + }); + + test('it throws error', async () => { + const storageResult = { indexName: 'test' } as unknown as StorageResult; + fetch.mockRejectedValueOnce('test-error'); + await postStorageResult({ + httpFetch: fetch, + storageResult, + abortController: new AbortController(), + toasts, + }); + expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); + }); +}); + +describe('getStorageResults', () => { + const { fetch } = httpServiceMock.createStartContract(); + const { toasts } = notificationServiceMock.createStartContract(); + beforeEach(() => { + fetch.mockClear(); + }); + + test('it gets the results', async () => { + await getStorageResults({ + httpFetch: fetch, + abortController: new AbortController(), + pattern: 'auditbeat-*', + toasts, + }); + + expect(fetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results_latest/auditbeat-*', + expect.objectContaining({ + method: 'GET', + }) + ); + }); + + it('should catch error', async () => { + fetch.mockRejectedValueOnce('test-error'); + + const results = await getStorageResults({ + httpFetch: fetch, + abortController: new AbortController(), + pattern: 'auditbeat-*', + toasts, + }); + + expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); + expect(results).toEqual([]); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts new file mode 100644 index 0000000000000..b7b3b120441d3 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpHandler } from '@kbn/core-http-browser'; +import { IToasts } from '@kbn/core-notifications-browser'; + +import { + DataQualityCheckResult, + DataQualityIndexCheckedParams, + IncompatibleFieldMappingItem, + IncompatibleFieldValueItem, + PartitionedFieldMetadata, + SameFamilyFieldItem, +} from '../../../types'; +import { StorageResult } from '../types'; +import { GET_INDEX_RESULTS_LATEST, POST_INDEX_RESULTS } from '../constants'; +import { INTERNAL_API_VERSION } from '../../../constants'; +import { GET_RESULTS_ERROR_TITLE, POST_RESULT_ERROR_TITLE } from '../../../translations'; + +export const formatStorageResult = ({ + result, + report, + partitionedFieldMetadata, +}: { + result: DataQualityCheckResult; + report: DataQualityIndexCheckedParams; + partitionedFieldMetadata: PartitionedFieldMetadata; +}): StorageResult => { + const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = []; + const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = []; + const sameFamilyFieldItems: SameFamilyFieldItem[] = []; + + partitionedFieldMetadata.incompatible.forEach((field) => { + if (field.type !== field.indexFieldType) { + incompatibleFieldMappingItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.type, + actualValue: field.indexFieldType, + description: field.description, + }); + } + + if (field.indexInvalidValues.length > 0) { + incompatibleFieldValueItems.push({ + fieldName: field.indexFieldName, + expectedValues: field.allowed_values?.map((x) => x.name) ?? [], + actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })), + description: field.description, + }); + } + }); + + partitionedFieldMetadata.sameFamily.forEach((field) => { + sameFamilyFieldItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.type, + actualValue: field.indexFieldType, + description: field.description, + }); + }); + + return { + batchId: report.batchId, + indexName: result.indexName, + indexPattern: result.pattern, + isCheckAll: report.isCheckAll, + checkedAt: result.checkedAt ?? Date.now(), + docsCount: result.docsCount ?? 0, + totalFieldCount: partitionedFieldMetadata.all.length, + ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, + customFieldCount: partitionedFieldMetadata.custom.length, + incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, + incompatibleFieldMappingItems, + incompatibleFieldValueItems, + sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, + sameFamilyFields: report.sameFamilyFields ?? [], + sameFamilyFieldItems, + unallowedMappingFields: report.unallowedMappingFields ?? [], + unallowedValueFields: report.unallowedValueFields ?? [], + sizeInBytes: report.sizeInBytes ?? 0, + ilmPhase: result.ilmPhase, + markdownComments: result.markdownComments, + ecsVersion: report.ecsVersion, + indexId: report.indexId ?? '', + error: result.error, + }; +}; + +export const formatResultFromStorage = ({ + storageResult, + pattern, +}: { + storageResult: StorageResult; + pattern: string; +}): DataQualityCheckResult => ({ + docsCount: storageResult.docsCount, + error: storageResult.error, + ilmPhase: storageResult.ilmPhase, + incompatible: storageResult.incompatibleFieldCount, + indexName: storageResult.indexName, + markdownComments: storageResult.markdownComments, + sameFamily: storageResult.sameFamilyFieldCount, + checkedAt: storageResult.checkedAt, + pattern, +}); + +export async function postStorageResult({ + storageResult, + httpFetch, + toasts, + abortController = new AbortController(), +}: { + storageResult: StorageResult; + httpFetch: HttpHandler; + toasts: IToasts; + abortController?: AbortController; +}): Promise { + try { + await httpFetch(POST_INDEX_RESULTS, { + method: 'POST', + signal: abortController.signal, + version: INTERNAL_API_VERSION, + body: JSON.stringify(storageResult), + }); + } catch (err) { + toasts.addError(err, { title: POST_RESULT_ERROR_TITLE }); + } +} + +export async function getStorageResults({ + pattern, + httpFetch, + toasts, + abortController, +}: { + pattern: string; + httpFetch: HttpHandler; + toasts: IToasts; + abortController: AbortController; +}): Promise { + try { + const route = GET_INDEX_RESULTS_LATEST.replace('{pattern}', pattern); + const results = await httpFetch(route, { + method: 'GET', + signal: abortController.signal, + version: INTERNAL_API_VERSION, + }); + return results; + } catch (err) { + toasts.addError(err, { title: GET_RESULTS_ERROR_TITLE }); + return []; + } +} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx index e27bdd26540e9..7d1a106d83570 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx @@ -13,13 +13,12 @@ import React, { useCallback, useMemo, useState } from 'react'; import type { IToasts } from '@kbn/core-notifications-browser'; import { EuiComboBoxOptionOption, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { DataQualityProvider } from './data_quality_context'; -import { EMPTY_STAT } from './helpers'; import { ReportDataQualityCheckAllCompleted, ReportDataQualityIndexChecked } from './types'; import { ResultsRollupContext } from './contexts/results_rollup_context'; import { IndicesCheckContext } from './contexts/indices_check_context'; import { useIndicesCheck } from './hooks/use_indices_check'; import { useResultsRollup } from './hooks/use_results_rollup'; -import { ilmPhaseOptionsStatic } from './constants'; +import { ilmPhaseOptionsStatic, EMPTY_STAT } from './constants'; import { DataQualitySummary } from './data_quality_summary'; import { DataQualityDetails } from './data_quality_details'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts index ac29ac9ac2fd1..264198e510b5e 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts @@ -8,7 +8,7 @@ import numeral from '@elastic/numeral'; import { DataQualityProviderProps } from '../../../data_quality_context'; -import { EMPTY_STAT } from '../../../helpers'; +import { EMPTY_STAT } from '../../../constants'; export const getMergedDataQualityContextProps = ( dataQualityContextProps?: Partial diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stats_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stats_rollup/index.tsx index 47eed189cbd31..7a0fb2deaf5bb 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stats_rollup/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stats_rollup/index.tsx @@ -9,10 +9,11 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React from 'react'; import styled from 'styled-components'; -import { EMPTY_STAT, getIncompatibleStatBadgeColor } from '../helpers'; +import { EMPTY_STAT } from '../constants'; import { useDataQualityContext } from '../data_quality_context'; import * as i18n from '../stat_label/translations'; import { Stat } from '../stat'; +import { getIncompatibleStatBadgeColor } from '../utils/get_incompatible_stat_badge_color'; const StyledStatWrapperFlexItem = styled(EuiFlexItem)` padding: 0 ${({ theme }) => theme.eui.euiSize}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/translations.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/translations.ts index 6c78513274de8..3497aa89d97ee 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/translations.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/translations.ts @@ -42,13 +42,6 @@ export const COLD_DESCRIPTION = i18n.translate( } ); -export const COLD_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) => - i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.coldPatternTooltip', { - values: { indices, pattern }, - defaultMessage: - '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.', - }); - export const COPIED_RESULTS_TOAST_TITLE = i18n.translate( 'securitySolutionPackages.ecsDataQualityDashboard.toasts.copiedResultsToastTitle', { @@ -166,18 +159,6 @@ export const FROZEN_DESCRIPTION = i18n.translate( } ); -export const FROZEN_PATTERN_TOOLTIP = ({ - indices, - pattern, -}: { - indices: number; - pattern: string; -}) => - i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.frozenPatternTooltip', { - values: { indices, pattern }, - defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.`, - }); - export const HOT_DESCRIPTION = i18n.translate( 'securitySolutionPackages.ecsDataQualityDashboard.hotDescription', { @@ -185,13 +166,6 @@ export const HOT_DESCRIPTION = i18n.translate( } ); -export const HOT_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) => - i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.hotPatternTooltip', { - values: { indices, pattern }, - defaultMessage: - '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} hot. Hot indices are actively being updated and queried.', - }); - /** The tooltip for the `ILM phase` combo box on the Data Quality Dashboard */ export const INDEX_LIFECYCLE_MANAGEMENT_PHASES: string = i18n.translate( 'securitySolutionPackages.ecsDataQualityDashboard.indexLifecycleManagementPhasesTooltip', @@ -267,18 +241,6 @@ export const UNMANAGED_DESCRIPTION = i18n.translate( } ); -export const UNMANAGED_PATTERN_TOOLTIP = ({ - indices, - pattern, -}: { - indices: number; - pattern: string; -}) => - i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.unmanagedPatternTooltip', { - values: { indices, pattern }, - defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} unmanaged by Index Lifecycle Management (ILM)`, - }); - export const WARM_DESCRIPTION = i18n.translate( 'securitySolutionPackages.ecsDataQualityDashboard.warmDescription', { @@ -286,13 +248,6 @@ export const WARM_DESCRIPTION = i18n.translate( } ); -export const WARM_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) => - i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.warmPatternTooltip', { - values: { indices, pattern }, - defaultMessage: - '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} warm. Warm indices are no longer being updated but are still being queried.', - }); - export const POST_RESULT_ERROR_TITLE = i18n.translate( 'securitySolutionPackages.ecsDataQualityDashboard.postResultErrorTitle', { defaultMessage: 'Error writing saved data quality check results' } diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/types.ts index dd5dd6dd0a159..916d0f377bf85 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/types.ts @@ -85,14 +85,6 @@ export interface PartitionedFieldMetadata { sameFamily: EcsBasedFieldMetadata[]; } -export interface PartitionedFieldMetadataStats { - all: number; - custom: number; - ecsCompliant: number; - incompatible: number; - sameFamily: number; -} - export interface UnallowedValueRequestItem { allowedValues: string[]; indexFieldName: string; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/check_index.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/check_index.test.ts index 512d26f365238..2b90f67966c77 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/check_index.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/check_index.test.ts @@ -6,7 +6,6 @@ */ import { checkIndex, EMPTY_PARTITIONED_FIELD_METADATA } from './check_index'; -import { EMPTY_STAT } from '../helpers'; import { mockMappingsResponse } from '../mock/mappings_response/mock_mappings_response'; import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values'; import { UnallowedValueRequestItem, UnallowedValueSearchResult } from '../types'; @@ -17,7 +16,7 @@ import { import { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types'; import { getUnallowedValues } from './fetch_unallowed_values'; import { getUnallowedValueRequestItems } from './get_unallowed_value_request_items'; -import { EcsFlatTyped } from '../constants'; +import { EcsFlatTyped, EMPTY_STAT } from '../constants'; let mockFetchMappings = jest.fn( (_: { abortController: AbortController; patternOrIndexName: string }) => diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_mappings.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_mappings.ts index 6bee012883c05..090f7e375e5bc 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_mappings.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_mappings.ts @@ -9,7 +9,7 @@ import type { HttpHandler } from '@kbn/core-http-browser'; import type { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types'; import * as i18n from '../translations'; -import { INTERNAL_API_VERSION } from '../helpers'; +import { INTERNAL_API_VERSION } from '../constants'; export const MAPPINGS_API_ROUTE = '/internal/ecs_data_quality_dashboard/mappings'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.test.ts index 9ceb2869bfa42..00dc04dedf27a 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.test.ts @@ -15,7 +15,7 @@ import { } from './fetch_unallowed_values'; import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values'; import { UnallowedValueRequestItem, UnallowedValueSearchResult } from '../types'; -import { INTERNAL_API_VERSION } from '../helpers'; +import { INTERNAL_API_VERSION } from '../constants'; describe('helpers', () => { let originalFetch: (typeof global)['fetch']; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.ts index aa25a5fe00b44..b19481546a285 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.ts @@ -6,7 +6,8 @@ */ import type { HttpHandler } from '@kbn/core-http-browser'; -import { INTERNAL_API_VERSION } from '../helpers'; + +import { INTERNAL_API_VERSION } from '../constants'; import * as i18n from '../translations'; import type { Bucket, @@ -15,8 +16,6 @@ import type { UnallowedValueSearchResult, } from '../types'; -const UNALLOWED_VALUES_API_ROUTE = '/internal/ecs_data_quality_dashboard/unallowed_field_values'; - export const isBucket = (maybeBucket: unknown): maybeBucket is Bucket => maybeBucket != null && typeof (maybeBucket as Bucket).key === 'string' && @@ -65,6 +64,7 @@ export const getUnallowedValues = ({ }, {}); }; +const UNALLOWED_VALUES_API_ROUTE = '/internal/ecs_data_quality_dashboard/unallowed_field_values'; export async function fetchUnallowedValues({ abortController, httpFetch, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.test.ts new file mode 100644 index 0000000000000..b2ff66941b395 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getIlmPhaseDescription } from './get_ilm_phase_description'; +import { + COLD_DESCRIPTION, + FROZEN_DESCRIPTION, + HOT_DESCRIPTION, + UNMANAGED_DESCRIPTION, + WARM_DESCRIPTION, +} from '../translations'; + +describe('helpers', () => { + describe('getIlmPhaseDescription', () => { + const phases: Array<{ + phase: string; + expected: string; + }> = [ + { + phase: 'hot', + expected: HOT_DESCRIPTION, + }, + { + phase: 'warm', + expected: WARM_DESCRIPTION, + }, + { + phase: 'cold', + expected: COLD_DESCRIPTION, + }, + { + phase: 'frozen', + expected: FROZEN_DESCRIPTION, + }, + { + phase: 'unmanaged', + expected: UNMANAGED_DESCRIPTION, + }, + { + phase: 'something-else', + expected: ' ', + }, + ]; + + phases.forEach(({ phase, expected }) => { + test(`it returns ${expected} when phase is ${phase}`, () => { + expect(getIlmPhaseDescription(phase)).toBe(expected); + }); + }); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.ts new file mode 100644 index 0000000000000..27f20cd28e1a4 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as i18n from '../translations'; + +/** + * Returns an i18n description of an an ILM phase + */ +export const getIlmPhaseDescription = (phase: string): string => { + switch (phase) { + case 'hot': + return i18n.HOT_DESCRIPTION; + case 'warm': + return i18n.WARM_DESCRIPTION; + case 'cold': + return i18n.COLD_DESCRIPTION; + case 'frozen': + return i18n.FROZEN_DESCRIPTION; + case 'unmanaged': + return i18n.UNMANAGED_DESCRIPTION; + default: + return ' '; + } +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.test.ts new file mode 100644 index 0000000000000..44283436f7477 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.test.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getIncompatibleStatBadgeColor } from './get_incompatible_stat_badge_color'; + +describe('getIncompatibleStatBadgeColor', () => { + describe('when incompatible is greater than 0', () => { + it('returns danger', () => { + expect(getIncompatibleStatBadgeColor(1)).toBe('danger'); + }); + }); + + describe('when incompatible is 0', () => { + it('returns hollow', () => { + expect(getIncompatibleStatBadgeColor(0)).toBe('hollow'); + }); + }); + + describe('when incompatible is undefined', () => { + it('returns hollow', () => { + expect(getIncompatibleStatBadgeColor(undefined)).toBe('hollow'); + }); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.ts new file mode 100644 index 0000000000000..fcaa660bba934 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getIncompatibleStatBadgeColor = (incompatible: number | undefined): string => + incompatible != null && incompatible > 0 ? 'danger' : 'hollow'; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.test.ts new file mode 100644 index 0000000000000..ad91466634ea7 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + auditbeatNoResults, + auditbeatWithAllResults, +} from '../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; +import { packetbeatWithSomeErrors } from '../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; +import { mockStatsPacketbeatIndex } from '../mock/stats/mock_stats_auditbeat_index'; +import { mockStatsAuditbeatIndex } from '../mock/stats/mock_stats_packetbeat_index'; +import { DataQualityCheckResult } from '../types'; +import { + getDocsCount, + getSizeInBytes, + getTotalPatternIncompatible, + getTotalPatternIndicesChecked, +} from './stats'; + +describe('getTotalPatternIndicesChecked', () => { + test('it returns zero when `patternRollup` is undefined', () => { + expect(getTotalPatternIndicesChecked(undefined)).toEqual(0); + }); + + test('it returns zero when `patternRollup` does NOT have any results', () => { + expect(getTotalPatternIndicesChecked(auditbeatNoResults)).toEqual(0); + }); + + test('it returns the expected total when all indices in `patternRollup` have results', () => { + expect(getTotalPatternIndicesChecked(auditbeatWithAllResults)).toEqual(3); + }); + + test('it returns the expected total when some indices in `patternRollup` have errors', () => { + expect(getTotalPatternIndicesChecked(packetbeatWithSomeErrors)).toEqual(1); + }); +}); + +describe('getDocsCount', () => { + test('it returns the expected docs count when `stats` contains the `indexName`', () => { + const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; + const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs; + + expect( + getDocsCount({ + indexName, + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(expectedCount); + }); + + test('it returns zero when `stats` does NOT contain the `indexName`', () => { + const indexName = 'not-gonna-find-it'; + + expect( + getDocsCount({ + indexName, + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(0); + }); + + test('it returns zero when `stats` is null', () => { + const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; + + expect( + getDocsCount({ + indexName, + stats: null, + }) + ).toEqual(0); + }); + + test('it returns the expected total for a green index, where `primaries.docs.count` and `total.docs.count` have different values', () => { + const indexName = 'auditbeat-custom-index-1'; + + expect( + getDocsCount({ + indexName, + stats: mockStatsAuditbeatIndex, + }) + ).toEqual(mockStatsAuditbeatIndex[indexName].num_docs); + }); +}); + +describe('getSizeInBytes', () => { + test('it returns the expected size when `stats` contains the `indexName`', () => { + const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; + const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes; + + expect( + getSizeInBytes({ + indexName, + stats: mockStatsPacketbeatIndex, + }) + ).toEqual(expectedCount); + }); + + test('it returns undefined when `stats` does NOT contain the `indexName`', () => { + const indexName = 'not-gonna-find-it'; + + expect( + getSizeInBytes({ + indexName, + stats: mockStatsPacketbeatIndex, + }) + ).toBeUndefined(); + }); + + test('it returns undefined when `stats` is null', () => { + const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; + + expect( + getSizeInBytes({ + indexName, + stats: null, + }) + ).toBeUndefined(); + }); + + test('it returns the expected size for a green index, where `primaries.store.size_in_bytes` and `total.store.size_in_bytes` have different values', () => { + const indexName = 'auditbeat-custom-index-1'; + + expect( + getSizeInBytes({ + indexName, + stats: mockStatsAuditbeatIndex, + }) + ).toEqual(mockStatsAuditbeatIndex[indexName].size_in_bytes); + }); +}); + +describe('getTotalPatternIncompatible', () => { + test('it returns zero when multiple indices in the results results have a count of zero', () => { + const results: Record = { + '.ds-packetbeat-8.5.3-2023.02.04-000001': { + docsCount: 1630289, + error: null, + ilmPhase: 'hot', + incompatible: 0, + indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'packetbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + '.ds-packetbeat-8.6.1-2023.02.04-000001': { + docsCount: 1628343, + error: null, + ilmPhase: 'hot', + incompatible: 0, + indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'packetbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + }; + + expect(getTotalPatternIncompatible(results)).toEqual(0); + }); + + test("it returns the expected total when some indices have incompatible fields, but others don't", () => { + const results: Record = { + '.ds-auditbeat-8.6.1-2023.02.07-000001': { + docsCount: 18086, + error: null, + ilmPhase: 'hot', + incompatible: 0, + indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + 'auditbeat-custom-index-1': { + docsCount: 4, + error: null, + ilmPhase: 'unmanaged', + incompatible: 3, + indexName: 'auditbeat-custom-index-1', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + 'auditbeat-custom-empty-index-1': { + docsCount: 0, + error: null, + ilmPhase: 'unmanaged', + incompatible: 1, + indexName: 'auditbeat-custom-empty-index-1', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + }; + + expect(getTotalPatternIncompatible(results)).toEqual(4); + }); + + test('it returns the expected total when some indices have undefined incompatible counts', () => { + const results: Record = { + '.ds-auditbeat-8.6.1-2023.02.07-000001': { + docsCount: 18086, + error: null, + ilmPhase: 'hot', + incompatible: undefined, // <-- this index has an undefined `incompatible` + indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + 'auditbeat-custom-index-1': { + docsCount: 4, + error: null, + ilmPhase: 'unmanaged', + incompatible: 3, + indexName: 'auditbeat-custom-index-1', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + 'auditbeat-custom-empty-index-1': { + docsCount: 0, + error: null, + ilmPhase: 'unmanaged', + incompatible: 1, + indexName: 'auditbeat-custom-empty-index-1', + markdownComments: ['foo', 'bar', 'baz'], + pattern: 'auditbeat-*', + sameFamily: 0, + checkedAt: Date.now(), + }, + }; + + expect(getTotalPatternIncompatible(results)).toEqual(4); + }); + + test('it returns zero when `results` is empty', () => { + expect(getTotalPatternIncompatible({})).toEqual(0); + }); + + test('it returns undefined when `results` is undefined', () => { + expect(getTotalPatternIncompatible(undefined)).toBeUndefined(); + }); +}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.ts new file mode 100644 index 0000000000000..b8f60be24a87c --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../types'; + +export const getSizeInBytes = ({ + indexName, + stats, +}: { + indexName: string; + stats: Record | null; +}): number | undefined => (stats && stats[indexName]?.size_in_bytes) ?? undefined; + +export const getDocsCount = ({ + indexName, + stats, +}: { + indexName: string; + stats: Record | null; +}): number => (stats && stats[indexName]?.num_docs) ?? 0; + +export const getTotalPatternIndicesChecked = (patternRollup: PatternRollup | undefined): number => { + if (patternRollup != null && patternRollup.results != null) { + const allResults = Object.values(patternRollup.results); + const nonErrorResults = allResults.filter(({ error }) => error == null); + + return nonErrorResults.length; + } else { + return 0; + } +}; + +export const getTotalPatternIncompatible = ( + results: Record | undefined +): number | undefined => { + if (results == null) { + return undefined; + } + + const allResults = Object.values(results); + + return allResults.reduce((acc, { incompatible }) => acc + (incompatible ?? 0), 0); +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/index.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/index.ts index 1ed64e65a74a6..96288f2bfec6c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/index.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/index.ts @@ -7,7 +7,7 @@ export { DataQualityPanel } from './impl/data_quality_panel'; -export { getIlmPhaseDescription } from './impl/data_quality_panel/helpers'; +export { getIlmPhaseDescription } from './impl/data_quality_panel/utils/get_ilm_phase_description'; export { DATA_QUALITY_PROMPT_CONTEXT_PILL,