diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.test.tsx index fb8eccd4b7f8a..a82bf7d6c432b 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.test.tsx @@ -65,6 +65,8 @@ const ContextWrapper: FC> = ({ children }) => ( }, ]} setSelectedIlmPhaseOptions={jest.fn()} + defaultStartTime="now-7d" + defaultEndTime="now" > {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.tsx index 762efef424a10..876ff528e75ff 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_context/index.tsx @@ -41,6 +41,8 @@ export interface DataQualityProviderProps { ilmPhases: string[]; selectedIlmPhaseOptions: EuiComboBoxOptionOption[]; setSelectedIlmPhaseOptions: (options: EuiComboBoxOptionOption[]) => void; + defaultStartTime: string; + defaultEndTime: string; } const DataQualityContext = React.createContext(undefined); @@ -67,6 +69,8 @@ export const DataQualityProvider: React.FC { const value = useMemo( () => ({ @@ -90,6 +94,8 @@ export const DataQualityProvider: React.FC {children} @@ -159,6 +161,8 @@ describe('useIlmExplain', () => { }, ]} setSelectedIlmPhaseOptions={jest.fn()} + defaultStartTime={'now-7d'} + defaultEndTime={'now'} > {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.test.tsx index 061bbb5aa6824..ae4ee9a7bd2c4 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.test.tsx @@ -69,6 +69,8 @@ const ContextWrapper: FC> = ({ children }) => ( }, ]} setSelectedIlmPhaseOptions={jest.fn()} + defaultStartTime={'now-7d'} + defaultEndTime={'now'} > {children} @@ -119,6 +121,8 @@ const ContextWrapperILMNotAvailable: FC> = ({ childre }, ]} setSelectedIlmPhaseOptions={jest.fn()} + defaultStartTime={'now-7d'} + defaultEndTime={'now'} > {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx index d58bf3af39d58..5f90890eea693 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.test.tsx @@ -11,6 +11,10 @@ import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; import { getHistoricalResultStub } from '../../../../stub/get_historical_result_stub'; import { useStoredPatternResults } from '.'; +const startTime = 'now-7d'; +const endTime = 'now'; +const isILMAvailable = true; + describe('useStoredPatternResults', () => { const httpFetch = jest.fn(); const mockToasts = notificationServiceMock.createStartContract().toasts; @@ -21,7 +25,16 @@ describe('useStoredPatternResults', () => { describe('when patterns are empty', () => { it('should return an empty array and not call getStorageResults', () => { - const { result } = renderHook(() => useStoredPatternResults([], mockToasts, httpFetch)); + const { result } = renderHook(() => + useStoredPatternResults({ + patterns: [], + toasts: mockToasts, + httpFetch, + isILMAvailable, + startTime, + endTime, + }) + ); expect(result.current).toEqual([]); expect(httpFetch).not.toHaveBeenCalled(); @@ -45,7 +58,14 @@ describe('useStoredPatternResults', () => { }); const { result, waitFor } = renderHook(() => - useStoredPatternResults(patterns, mockToasts, httpFetch) + useStoredPatternResults({ + patterns, + toasts: mockToasts, + httpFetch, + isILMAvailable, + startTime, + endTime, + }) ); await waitFor(() => result.current.length > 0); @@ -104,5 +124,63 @@ describe('useStoredPatternResults', () => { }, ]); }); + + describe('when isILMAvailable is false', () => { + it('should call getStorageResults with startDate and endDate', async () => { + const patterns = ['pattern1-*', 'pattern2-*']; + + httpFetch.mockImplementation((path: string) => { + if (path === '/internal/ecs_data_quality_dashboard/results_latest/pattern1-*') { + return Promise.resolve([getHistoricalResultStub('pattern1-index1')]); + } + + if (path === '/internal/ecs_data_quality_dashboard/results_latest/pattern2-*') { + return Promise.resolve([getHistoricalResultStub('pattern2-index1')]); + } + + return Promise.reject(new Error('Invalid path')); + }); + + const { result, waitFor } = renderHook(() => + useStoredPatternResults({ + patterns, + toasts: mockToasts, + httpFetch, + isILMAvailable: false, + startTime, + endTime, + }) + ); + + await waitFor(() => result.current.length > 0); + + expect(httpFetch).toHaveBeenCalledTimes(2); + + expect(httpFetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results_latest/pattern1-*', + { + method: 'GET', + signal: expect.any(AbortSignal), + version: '1', + query: { + startDate: startTime, + endDate: endTime, + }, + } + ); + expect(httpFetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results_latest/pattern2-*', + { + method: 'GET', + signal: expect.any(AbortSignal), + version: '1', + query: { + startDate: startTime, + endDate: endTime, + }, + } + ); + }); + }); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx index 17334c4b4a586..b92b36218c07a 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/hooks/use_stored_pattern_results/index.tsx @@ -10,13 +10,34 @@ import { HttpHandler } from '@kbn/core-http-browser'; import { isEmpty } from 'lodash/fp'; import { DataQualityCheckResult } from '../../../../types'; -import { formatResultFromStorage, getStorageResults } from '../../utils/storage'; +import { + GetStorageResultsOpts, + formatResultFromStorage, + getStorageResults, +} from '../../utils/storage'; -export const useStoredPatternResults = ( - patterns: string[], - toasts: IToasts, - httpFetch: HttpHandler -) => { +export interface UseStoredPatternResultsOpts { + patterns: string[]; + toasts: IToasts; + httpFetch: HttpHandler; + isILMAvailable: boolean; + startTime: string; + endTime: string; +} + +export type UseStoredPatternResultsReturnValue = Array<{ + pattern: string; + results: Record; +}>; + +export const useStoredPatternResults = ({ + patterns, + toasts, + httpFetch, + isILMAvailable, + startTime, + endTime, +}: UseStoredPatternResultsOpts): UseStoredPatternResultsReturnValue => { const [storedPatternResults, setStoredPatternResults] = useState< Array<{ pattern: string; results: Record }> >([]); @@ -28,8 +49,20 @@ export const useStoredPatternResults = ( const abortController = new AbortController(); const fetchStoredPatternResults = async () => { - const requests = patterns.map((pattern) => - getStorageResults({ pattern, httpFetch, abortController, toasts }).then((results = []) => ({ + const requests = patterns.map(async (pattern) => { + const getStorageResultsOpts: GetStorageResultsOpts = { + pattern, + httpFetch, + abortController, + toasts, + }; + + if (!isILMAvailable) { + getStorageResultsOpts.startTime = startTime; + getStorageResultsOpts.endTime = endTime; + } + + return getStorageResults(getStorageResultsOpts).then((results) => ({ pattern, results: Object.fromEntries( results.map((storageResult) => [ @@ -37,8 +70,8 @@ export const useStoredPatternResults = ( formatResultFromStorage({ storageResult, pattern }), ]) ), - })) - ); + })); + }); const patternResults = await Promise.all(requests); if (patternResults?.length) { @@ -47,7 +80,7 @@ export const useStoredPatternResults = ( }; fetchStoredPatternResults(); - }, [httpFetch, patterns, toasts]); + }, [endTime, httpFetch, isILMAvailable, patterns, startTime, toasts]); return storedPatternResults; }; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx index bff3c3dd54f12..7dc74731d66dd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.test.tsx @@ -35,6 +35,8 @@ describe('useResultsRollup', () => { const patterns = ['auditbeat-*', 'packetbeat-*']; const isILMAvailable = true; + const startTime = 'now-7d'; + const endTime = 'now'; const useStoredPatternResultsMock = useStoredPatternResults as jest.Mock; @@ -52,6 +54,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -94,10 +98,19 @@ describe('useResultsRollup', () => { patterns: ['auditbeat-*'], isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); - expect(useStoredPatternResultsMock).toHaveBeenCalledWith(['auditbeat-*'], toasts, httpFetch); + expect(useStoredPatternResultsMock).toHaveBeenCalledWith({ + patterns: ['auditbeat-*'], + toasts, + httpFetch, + isILMAvailable, + startTime, + endTime, + }); expect(result.current.patternRollups).toEqual({ 'auditbeat-*': { @@ -119,6 +132,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -144,6 +159,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -180,6 +197,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -369,6 +388,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable: false, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -532,6 +553,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -592,6 +615,8 @@ describe('useResultsRollup', () => { patterns, isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); @@ -654,6 +679,8 @@ describe('useResultsRollup', () => { patterns: ['packetbeat-*', 'auditbeat-*'], isILMAvailable, telemetryEvents: mockTelemetryEvents, + startTime, + endTime, }) ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx index d95f1d1b7f20f..bfed849e373d3 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx @@ -40,6 +40,8 @@ interface Props { httpFetch: HttpHandler; telemetryEvents: TelemetryEvents; isILMAvailable: boolean; + startTime: string; + endTime: string; } export const useResultsRollup = ({ httpFetch, @@ -47,11 +49,20 @@ export const useResultsRollup = ({ patterns, isILMAvailable, telemetryEvents, + startTime, + endTime, }: Props): UseResultsRollupReturnValue => { const [patternIndexNames, setPatternIndexNames] = useState>({}); const [patternRollups, setPatternRollups] = useState>({}); - const storedPatternsResults = useStoredPatternResults(patterns, toasts, httpFetch); + const storedPatternsResults = useStoredPatternResults({ + httpFetch, + patterns, + toasts, + isILMAvailable, + startTime, + endTime, + }); useEffect(() => { if (!isEmpty(storedPatternsResults)) { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts index 9f315d65c01d5..b43954e73f6fd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts @@ -200,4 +200,26 @@ describe('getStorageResults', () => { expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); expect(results).toEqual([]); }); + + it('should provide stad and end date', async () => { + await getStorageResults({ + httpFetch: fetch, + abortController: new AbortController(), + pattern: 'auditbeat-*', + toasts, + startTime: 'now-7d', + endTime: 'now', + }); + + expect(fetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results_latest/auditbeat-*', + expect.objectContaining({ + method: 'GET', + query: { + startDate: 'now-7d', + endDate: 'now', + }, + }) + ); + }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts index e4a5c43d5b4a5..7fc339c085bea 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { HttpHandler } from '@kbn/core-http-browser'; +import { HttpFetchQuery, HttpHandler } from '@kbn/core-http-browser'; import { IToasts } from '@kbn/core-notifications-browser'; import { @@ -131,23 +131,40 @@ export async function postStorageResult({ } } +export interface GetStorageResultsOpts { + pattern: string; + httpFetch: HttpHandler; + toasts: IToasts; + abortController: AbortController; + startTime?: string; + endTime?: string; +} + export async function getStorageResults({ pattern, httpFetch, toasts, abortController, -}: { - pattern: string; - httpFetch: HttpHandler; - toasts: IToasts; - abortController: AbortController; -}): Promise { + startTime, + endTime, +}: GetStorageResultsOpts): Promise { try { const route = GET_INDEX_RESULTS_LATEST.replace('{pattern}', pattern); + + const query: HttpFetchQuery = {}; + + if (startTime) { + query.startDate = startTime; + } + if (endTime) { + query.endDate = endTime; + } + const results = await httpFetch(route, { method: 'GET', signal: abortController.signal, version: INTERNAL_API_VERSION, + ...(Object.keys(query).length > 0 ? { query } : {}), }); return results; } catch (err) { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.test.tsx index 90e5dba08d4dc..f925a67ea3d32 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.test.tsx @@ -67,6 +67,8 @@ describe('DataQualityPanel', () => { setLastChecked={jest.fn()} baseTheme={DARK_THEME} toasts={toasts} + defaultStartTime={'now-7d'} + defaultEndTime={'now'} /> ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx index b6d2736d7e175..9b9cbdefb6670 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx @@ -46,6 +46,8 @@ interface Props { setLastChecked: (lastChecked: string) => void; startDate?: string | null; theme?: PartialTheme; + defaultStartTime: string; + defaultEndTime: string; } const defaultSelectedIlmPhaseOptions: EuiComboBoxOptionOption[] = ilmPhaseOptionsStatic.filter( @@ -71,6 +73,8 @@ const DataQualityPanelComponent: React.FC = ({ setLastChecked, startDate, theme, + defaultStartTime, + defaultEndTime, }) => { const [selectedIlmPhaseOptions, setSelectedIlmPhaseOptions] = useState( defaultSelectedIlmPhaseOptions @@ -109,6 +113,8 @@ const DataQualityPanelComponent: React.FC = ({ toasts, isILMAvailable, telemetryEvents, + startTime: defaultStartTime, + endTime: defaultEndTime, }); const indicesCheckHookReturnValue = useIndicesCheck({ @@ -137,6 +143,8 @@ const DataQualityPanelComponent: React.FC = ({ ilmPhases={ilmPhases} selectedIlmPhaseOptions={selectedIlmPhaseOptions} setSelectedIlmPhaseOptions={setSelectedIlmPhaseOptions} + defaultStartTime={defaultStartTime} + defaultEndTime={defaultEndTime} > diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/test_providers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/test_providers.tsx index 17b73f1e6dcd0..e0220d26e8690 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/test_providers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/test_providers.tsx @@ -135,6 +135,8 @@ const TestDataQualityProvidersComponent: React.FC ilmPhases, selectedIlmPhaseOptions, setSelectedIlmPhaseOptions, + defaultStartTime, + defaultEndTime, } = getMergedDataQualityContextProps(dataQualityContextProps); const mergedResultsRollupContextProps = @@ -162,6 +164,8 @@ const TestDataQualityProvidersComponent: React.FC ilmPhases={ilmPhases} selectedIlmPhaseOptions={selectedIlmPhaseOptions} setSelectedIlmPhaseOptions={setSelectedIlmPhaseOptions} + defaultStartTime={defaultStartTime} + defaultEndTime={defaultEndTime} > ({ - index: indexPattern, + index: indexNameOrPattern, aggs: { index: { terms: { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_range_filtered_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_range_filtered_indices.test.ts new file mode 100644 index 0000000000000..87350abcf8a9c --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_range_filtered_indices.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRangeFilteredIndices } from './get_range_filtered_indices'; +import { fetchAvailableIndices } from '../lib/fetch_available_indices'; +import type { IScopedClusterClient, Logger } from '@kbn/core/server'; + +jest.mock('../lib/fetch_available_indices'); + +const fetchAvailableIndicesMock = fetchAvailableIndices as jest.Mock; + +describe('getRangeFilteredIndices', () => { + let client: jest.Mocked; + let logger: jest.Mocked; + + beforeEach(() => { + client = { + asCurrentUser: jest.fn(), + } as unknown as jest.Mocked; + + logger = { + warn: jest.fn(), + error: jest.fn(), + } as unknown as jest.Mocked; + + jest.clearAllMocks(); + }); + + describe('when fetching available indices is successful', () => { + describe('and there are available indices', () => { + it('should return the flattened available indices', async () => { + fetchAvailableIndicesMock.mockResolvedValueOnce(['index1', 'index2']); + fetchAvailableIndicesMock.mockResolvedValueOnce(['index3']); + + const result = await getRangeFilteredIndices({ + client, + authorizedIndexNames: ['auth1', 'auth2'], + startDate: '2023-01-01', + endDate: '2023-01-31', + logger, + pattern: 'pattern*', + }); + + expect(fetchAvailableIndices).toHaveBeenCalledTimes(2); + expect(result).toEqual(['index1', 'index2', 'index3']); + expect(logger.warn).not.toHaveBeenCalled(); + }); + }); + + describe('and there are no available indices', () => { + it('should log a warning and return an empty array', async () => { + fetchAvailableIndicesMock.mockResolvedValue([]); + + const result = await getRangeFilteredIndices({ + client, + authorizedIndexNames: ['auth1', 'auth2'], + startDate: '2023-01-01', + endDate: '2023-01-31', + logger, + pattern: 'pattern*', + }); + + expect(fetchAvailableIndices).toHaveBeenCalledTimes(2); + expect(result).toEqual([]); + expect(logger.warn).toHaveBeenCalledWith( + 'No available authorized indices found under pattern: pattern*, in the given date range: 2023-01-01 - 2023-01-31' + ); + }); + }); + }); + + describe('when fetching available indices fails', () => { + it('should log an error and return an empty array', async () => { + fetchAvailableIndicesMock.mockRejectedValue(new Error('Fetch error')); + + const result = await getRangeFilteredIndices({ + client, + authorizedIndexNames: ['auth1'], + startDate: '2023-01-01', + endDate: '2023-01-31', + logger, + pattern: 'pattern*', + }); + + expect(fetchAvailableIndices).toHaveBeenCalledTimes(1); + expect(result).toEqual([]); + expect(logger.error).toHaveBeenCalledWith( + 'Error fetching available indices in the given data range: 2023-01-01 - 2023-01-31' + ); + }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_range_filtered_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_range_filtered_indices.ts new file mode 100644 index 0000000000000..45a87424169e8 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_range_filtered_indices.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IScopedClusterClient, Logger } from '@kbn/core/server'; + +import { fetchAvailableIndices } from '../lib/fetch_available_indices'; + +export const getRangeFilteredIndices = async ({ + client, + authorizedIndexNames, + startDate, + endDate, + logger, + pattern, +}: { + client: IScopedClusterClient; + authorizedIndexNames: string[]; + startDate: string; + endDate: string; + logger: Logger; + pattern: string; +}): Promise => { + const decodedStartDate = decodeURIComponent(startDate); + const decodedEndDate = decodeURIComponent(endDate); + try { + const currentUserEsClient = client.asCurrentUser; + + const availableIndicesPromises: Array> = []; + + for (const indexName of authorizedIndexNames) { + availableIndicesPromises.push( + fetchAvailableIndices(currentUserEsClient, { + indexNameOrPattern: indexName, + startDate: decodedStartDate, + endDate: decodedEndDate, + }) + ); + } + + const availableIndices = await Promise.all(availableIndicesPromises); + + const flattenedAvailableIndices = availableIndices.flat(); + + if (flattenedAvailableIndices.length === 0) { + logger.warn( + `No available authorized indices found under pattern: ${pattern}, in the given date range: ${decodedStartDate} - ${decodedEndDate}` + ); + } + + return flattenedAvailableIndices; + } catch (err) { + logger.error( + `Error fetching available indices in the given data range: ${decodedStartDate} - ${decodedEndDate}` + ); + return []; + } +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts index fa26fb68289a6..9fe8213b4eb95 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts @@ -61,7 +61,7 @@ describe('fetchAvailableIndices', () => { const esClientMock = getEsClientMock(); await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -101,7 +101,7 @@ describe('fetchAvailableIndices', () => { const esClientMock = getEsClientMock(); await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -133,7 +133,7 @@ describe('fetchAvailableIndices', () => { ]); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -164,7 +164,7 @@ describe('fetchAvailableIndices', () => { ]); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -180,7 +180,7 @@ describe('fetchAvailableIndices', () => { esClientMock.cat.indices.mockResolvedValue([]); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'nonexistent-*', + indexNameOrPattern: 'nonexistent-*', startDate: startDateString, endDate: endDateString, }); @@ -209,7 +209,7 @@ describe('fetchAvailableIndices', () => { }); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -243,7 +243,7 @@ describe('fetchAvailableIndices', () => { }); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -268,7 +268,7 @@ describe('fetchAvailableIndices', () => { ]); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -285,7 +285,7 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }) @@ -307,7 +307,7 @@ describe('fetchAvailableIndices', () => { }); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -336,7 +336,7 @@ describe('fetchAvailableIndices', () => { }); const result = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }); @@ -371,7 +371,7 @@ describe('fetchAvailableIndices', () => { ]); const results = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: 'now-7d/d', endDate: 'now/d', }); @@ -390,7 +390,7 @@ describe('fetchAvailableIndices', () => { ]); const results = await fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: 'now-7d/d', endDate: 'now-1d/d', }); @@ -415,7 +415,7 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: endDateString, }) @@ -429,7 +429,7 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: 'invalid-date', endDate: endDateString, }) @@ -443,7 +443,7 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { - indexPattern: 'logs-*', + indexNameOrPattern: 'logs-*', startDate: startDateString, endDate: 'invalid-date', }) diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts index 32311f28d636a..36009f315010b 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts @@ -32,15 +32,15 @@ const getParsedDateMs = (dateStr: string, roundUp = false) => { export const fetchAvailableIndices = async ( esClient: ElasticsearchClient, - params: { indexPattern: string; startDate: string; endDate: string } + params: { indexNameOrPattern: string; startDate: string; endDate: string } ): Promise => { - const { indexPattern, startDate, endDate } = params; + const { indexNameOrPattern, startDate, endDate } = params; const startDateMs = getParsedDateMs(startDate); const endDateMs = getParsedDateMs(endDate, true); const indicesCats = (await esClient.cat.indices({ - index: indexPattern, + index: indexNameOrPattern, format: 'json', h: 'index,creation.date', })) as FetchAvailableCatIndicesResponseRequired; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index d1bb25d34fc2a..fd1ec1694719d 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -85,7 +85,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { const meteringStatsIndices = parseMeteringStats(meteringStats.indices); const availableIndices = await fetchAvailableIndices(esClient, { - indexPattern: decodedIndexName, + indexNameOrPattern: decodedIndexName, startDate: decodedStartDate, endDate: decodedEndDate, }); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.test.ts index bfb38864916fe..94c892e401b5a 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.test.ts @@ -16,6 +16,24 @@ import { resultDocument } from './results.mock'; import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import type { ResultDocument } from '../../schemas/result'; import type { CheckIndicesPrivilegesParam } from './privileges'; +import { getRangeFilteredIndices } from '../../helpers/get_range_filtered_indices'; + +const mockCheckIndicesPrivileges = jest.fn(({ indices }: CheckIndicesPrivilegesParam) => + Promise.resolve(Object.fromEntries(indices.map((index) => [index, true]))) +); +jest.mock('./privileges', () => ({ + checkIndicesPrivileges: (params: CheckIndicesPrivilegesParam) => + mockCheckIndicesPrivileges(params), +})); + +jest.mock('../../helpers/get_range_filtered_indices', () => ({ + getRangeFilteredIndices: jest.fn(), +})); + +const mockGetRangeFilteredIndices = getRangeFilteredIndices as jest.Mock; + +const startDate = 'now-7d'; +const endDate = 'now'; const searchResponse = { aggregations: { @@ -33,14 +51,6 @@ const searchResponse = { Record >; -const mockCheckIndicesPrivileges = jest.fn(({ indices }: CheckIndicesPrivilegesParam) => - Promise.resolve(Object.fromEntries(indices.map((index) => [index, true]))) -); -jest.mock('./privileges', () => ({ - checkIndicesPrivileges: (params: CheckIndicesPrivilegesParam) => - mockCheckIndicesPrivileges(params), -})); - describe('getIndexResultsLatestRoute route', () => { describe('querying', () => { let server: ReturnType; @@ -68,7 +78,7 @@ describe('getIndexResultsLatestRoute route', () => { getIndexResultsLatestRoute(server.router, logger); }); - it('gets result', async () => { + it('gets result without startDate and endDate', async () => { const mockSearch = context.core.elasticsearch.client.asInternalUser.search; mockSearch.mockResolvedValueOnce(searchResponse); @@ -80,6 +90,159 @@ describe('getIndexResultsLatestRoute route', () => { expect(response.status).toEqual(200); expect(response.body).toEqual([resultDocument]); + + expect(mockGetRangeFilteredIndices).not.toHaveBeenCalled(); + }); + + it('gets result with startDate and endDate', async () => { + const reqWithDate = requestMock.create({ + method: 'get', + path: GET_INDEX_RESULTS_LATEST, + params: { pattern: 'logs-*' }, + query: { startDate, endDate }, + }); + + const filteredIndices = ['filtered-index-1', 'filtered-index-2']; + mockGetRangeFilteredIndices.mockResolvedValueOnce(filteredIndices); + const mockSearch = context.core.elasticsearch.client.asInternalUser.search; + mockSearch.mockResolvedValueOnce(searchResponse); + + const response = await server.inject(reqWithDate, requestContextMock.convertContext(context)); + + expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + authorizedIndexNames: [resultDocument.indexName], + startDate, + endDate, + logger, + pattern: 'logs-*', + }); + + expect(mockSearch).toHaveBeenCalledWith({ + index: expect.any(String), + ...getQuery(filteredIndices), + }); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([resultDocument]); + }); + + it('handles getRangeFilteredIndices error', async () => { + const errorMessage = 'Range Filter Error'; + + const reqWithDate = requestMock.create({ + method: 'get', + path: GET_INDEX_RESULTS_LATEST, + params: { pattern: 'logs-*' }, + query: { startDate, endDate }, + }); + + mockGetRangeFilteredIndices.mockRejectedValueOnce(new Error(errorMessage)); + + const response = await server.inject(reqWithDate, requestContextMock.convertContext(context)); + + expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + authorizedIndexNames: [resultDocument.indexName], + startDate, + endDate, + logger, + pattern: 'logs-*', + }); + + expect(response.status).toEqual(500); + expect(response.body).toEqual({ message: errorMessage, status_code: 500 }); + expect(logger.error).toHaveBeenCalledWith(errorMessage); + }); + + it('gets result with startDate and endDate and multiple filtered indices', async () => { + const filteredIndices = ['filtered-index-1', 'filtered-index-2', 'filtered-index-3']; + const filteredIndicesSearchResponse = { + aggregations: { + latest: { + buckets: filteredIndices.map((indexName) => ({ + key: indexName, + latest_doc: { hits: { hits: [{ _source: { indexName } }] } }, + })), + }, + }, + } as unknown as SearchResponse< + ResultDocument, + Record + >; + + const reqWithDate = requestMock.create({ + method: 'get', + path: GET_INDEX_RESULTS_LATEST, + params: { pattern: 'logs-*' }, + query: { startDate, endDate }, + }); + + mockGetRangeFilteredIndices.mockResolvedValueOnce(filteredIndices); + context.core.elasticsearch.client.asInternalUser.search.mockResolvedValueOnce( + filteredIndicesSearchResponse + ); + + const response = await server.inject(reqWithDate, requestContextMock.convertContext(context)); + + expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + authorizedIndexNames: [resultDocument.indexName], + startDate, + endDate, + logger, + pattern: 'logs-*', + }); + + expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({ + index: expect.any(String), + ...getQuery(filteredIndices), + }); + + const expectedResults = filteredIndices.map((indexName) => ({ + indexName, + })) as ResultDocument[]; + expect(response.status).toEqual(200); + expect(response.body).toEqual(expectedResults); + }); + + it('handles partial authorization when using startDate and endDate', async () => { + const authorizationResult = { + 'filtered-index-1': true, + 'filtered-index-2': false, + }; + + mockGetRangeFilteredIndices.mockResolvedValueOnce(['filtered-index-1']); + mockCheckIndicesPrivileges.mockResolvedValueOnce(authorizationResult); + + const mockSearch = context.core.elasticsearch.client.asInternalUser.search; + mockSearch.mockResolvedValueOnce(searchResponse); + + const reqWithDate = requestMock.create({ + method: 'get', + path: GET_INDEX_RESULTS_LATEST, + params: { pattern: 'logs-*' }, + query: { startDate, endDate }, + }); + + const response = await server.inject(reqWithDate, requestContextMock.convertContext(context)); + + expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + authorizedIndexNames: ['filtered-index-1'], + startDate, + endDate, + logger, + pattern: 'logs-*', + }); + + expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({ + index: expect.any(String), + ...getQuery(['filtered-index-1']), + }); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([resultDocument]); }); it('handles results data stream error', async () => { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.ts index 3a294409af869..f7d1d5eed74cc 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_index_results_latest.ts @@ -4,18 +4,18 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import type { IRouter, Logger } from '@kbn/core/server'; import { INTERNAL_API_VERSION, GET_INDEX_RESULTS_LATEST } from '../../../common/constants'; import { buildResponse } from '../../lib/build_response'; import { buildRouteValidation } from '../../schemas/common'; -import { GetIndexResultsLatestParams } from '../../schemas/result'; +import { GetIndexResultsLatestParams, GetIndexResultsLatestQuery } from '../../schemas/result'; import type { ResultDocument } from '../../schemas/result'; import { API_DEFAULT_ERROR_MESSAGE } from '../../translations'; import type { DataQualityDashboardRequestHandlerContext } from '../../types'; import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations'; import { getAuthorizedIndexNames } from '../../helpers/get_authorized_index_names'; +import { getRangeFilteredIndices } from '../../helpers/get_range_filtered_indices'; export const getQuery = (indexName: string[]) => ({ size: 0, @@ -53,6 +53,7 @@ export const getIndexResultsLatestRoute = ( validate: { request: { params: buildRouteValidation(GetIndexResultsLatestParams), + query: buildRouteValidation(GetIndexResultsLatestQuery), }, }, }, @@ -81,8 +82,27 @@ export const getIndexResultsLatestRoute = ( return response.ok({ body: [] }); } + const { startDate, endDate } = request.query; + + let resultingIndices: string[] = []; + + if (startDate && endDate) { + resultingIndices = resultingIndices.concat( + await getRangeFilteredIndices({ + client, + authorizedIndexNames, + startDate, + endDate, + logger, + pattern, + }) + ); + } else { + resultingIndices = authorizedIndexNames; + } + // Get the latest result for each indexName - const query = { index, ...getQuery(authorizedIndexNames) }; + const query = { index, ...getQuery(resultingIndices) }; const { aggregations } = await client.asInternalUser.search< ResultDocument, Record diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts index 8ccb3fbc3f984..fb264fe10da8f 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts @@ -69,6 +69,11 @@ export const PostIndexResultBody = ResultDocument; export const GetIndexResultsLatestParams = t.type({ pattern: t.string }); export type GetIndexResultsLatestParams = t.TypeOf; +export const GetIndexResultsLatestQuery = t.partial({ + startDate: t.string, + endDate: t.string, +}); + export const GetIndexResultsParams = t.type({ pattern: t.string, }); diff --git a/x-pack/plugins/security_solution/public/overview/pages/data_quality.test.tsx b/x-pack/plugins/security_solution/public/overview/pages/data_quality.test.tsx index e39e2abd24169..8b14fff8082c5 100644 --- a/x-pack/plugins/security_solution/public/overview/pages/data_quality.test.tsx +++ b/x-pack/plugins/security_solution/public/overview/pages/data_quality.test.tsx @@ -8,6 +8,7 @@ import { render, screen, waitFor } from '@testing-library/react'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; +import type { HttpFetchOptions } from '@kbn/core-http-browser'; import { useKibana as mockUseKibana } from '../../common/lib/kibana/__mocks__'; import { TestProviders } from '../../common/mock'; @@ -22,7 +23,17 @@ jest.mock('../../common/lib/kibana', () => { const mockKibanaServices = { get: () => ({ - http: { fetch: jest.fn() }, + http: { + fetch: jest.fn().mockImplementation((path: string, options: HttpFetchOptions) => { + if ( + path.startsWith('/internal/ecs_data_quality_dashboard/results_latest') && + options.method === 'GET' + ) { + return Promise.resolve([]); + } + return Promise.resolve(); + }), + }, }), }; diff --git a/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx b/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx index 37fc927094993..67dcc3848f02a 100644 --- a/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx +++ b/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx @@ -171,6 +171,8 @@ const DataQualityComponent: React.FC = () => { startDate={startDate} theme={theme} toasts={toasts} + defaultStartTime={DEFAULT_START_TIME} + defaultEndTime={DEFAULT_END_TIME} /> ) : (