From 5eec1c9d7407db3478695d6b38cf94a0359e79da Mon Sep 17 00:00:00 2001 From: Karen Grigoryan Date: Mon, 21 Oct 2024 15:44:30 +0200 Subject: [PATCH 1/4] [Security Solution][Data Quality Dashboard][Serverless] Fix fetchAvailableIndices in get_index_stats addresses #196528 - Remove unused get_available_indices.ts params helper file. - Change fetchAvailableIndices to use creation_date from _cat api instead of targeting @timestamp field of indices --- .../server/helpers/get_available_indices.ts | 46 ----- .../lib/fetch_available_indices.test.ts | 183 ++++++++++++++++++ .../server/lib/fetch_available_indices.ts | 68 ++++++- 3 files changed, 243 insertions(+), 54 deletions(-) delete mode 100644 x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts create mode 100644 x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts deleted file mode 100644 index 8f7fdead51547..0000000000000 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; - -export const getRequestBody = ({ - indexPattern, - startDate = 'now-7d/d', - endDate = 'now/d', -}: { - indexPattern: string; - startDate: string; - endDate: string; -}): SearchRequest => ({ - index: indexPattern, - aggs: { - index: { - terms: { - field: '_index', - }, - }, - }, - size: 0, - query: { - bool: { - must: [], - filter: [ - { - range: { - '@timestamp': { - format: 'strict_date_optional_time', - gte: startDate, - lte: endDate, - }, - }, - }, - ], - should: [], - must_not: [], - }, - }, -}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts new file mode 100644 index 0000000000000..1e71ba0b090ad --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; + +import type { FetchAvailableCatIndicesResponseRequired } from './fetch_available_indices'; +import { fetchAvailableIndices } from './fetch_available_indices'; + +function getEsClientMock() { + return { + cat: { + indices: jest.fn().mockResolvedValue([]), + }, + } as unknown as ElasticsearchClient & { + cat: { + indices: jest.Mock>; + }; + }; +} + +const DAY_IN_MILLIS = 24 * 60 * 60 * 1000; + +describe('fetchAvailableIndices', () => { + const startDate: string = '2021-10-01'; + const endDate: string = '2021-10-07'; + + const startDateMillis: number = new Date(startDate).getTime(); + const endDateMillis: number = new Date(endDate).getTime(); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('when indices are within the date range', () => { + it('returns indices within the date range', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + { + index: 'logs-2021.09.30', + 'creation.date': `${startDateMillis - DAY_IN_MILLIS}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate, + endDate, + }); + + expect(result).toEqual({ + aggregations: { + index: { + buckets: [{ key: 'logs-2021.10.01' }, { key: 'logs-2021.10.05' }], + }, + }, + }); + + expect(esClientMock.cat.indices).toHaveBeenCalledWith({ + index: 'logs-*', + format: 'json', + h: 'index,creation.date', + }); + }); + }); + + describe('when indices are outside the date range', () => { + it('returns an empty list', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.09.30', + 'creation.date': `${startDateMillis - DAY_IN_MILLIS}`, + }, + { + index: 'logs-2021.10.08', + 'creation.date': `${endDateMillis + DAY_IN_MILLIS}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate, + endDate, + }); + + expect(result).toEqual({ + aggregations: { + index: { + buckets: [], + }, + }, + }); + }); + }); + + describe('when no indices match the index pattern', () => { + it('returns empty buckets', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'nonexistent-*', + startDate, + endDate, + }); + + expect(result).toEqual({ + aggregations: { + index: { + buckets: [], + }, + }, + }); + }); + }); + + describe('rejections', () => { + beforeEach(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { + jest.restoreAllMocks(); + }); + describe('when esClient.cat.indices rejects', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockRejectedValue(new Error('Elasticsearch error')); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate, + endDate, + }) + ).rejects.toThrow('Elasticsearch error'); + }); + }); + + describe('when startDate is invalid', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'invalid-date', + endDate, + }) + ).rejects.toThrow('Invalid date format in startDate or endDate'); + }); + }); + + describe('when endDate is invalid', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate, + endDate: 'invalid-date', + }) + ).rejects.toThrow('Invalid date format in startDate or endDate'); + }); + }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts index 584a261689113..7a1db0dea796f 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts @@ -4,18 +4,70 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import type { ElasticsearchClient } from '@kbn/core/server'; -import { getRequestBody } from '../helpers/get_available_indices'; +import type { CatIndicesIndicesRecord } from '@elastic/elasticsearch/lib/api/types'; +import dateMath from '@kbn/datemath'; + +export type FetchAvailableCatIndicesResponseRequired = Array< + Required> +>; -type AggregateName = 'index'; -interface Result { - index: { - buckets: Array<{ key: string }>; - doc_count: number; +export interface FetchAvailableIndicesResponse { + aggregations: { + index: { + buckets: Array<{ + key: string; + }>; + }; }; } -export const fetchAvailableIndices = ( +export const fetchAvailableIndices = async ( esClient: ElasticsearchClient, params: { indexPattern: string; startDate: string; endDate: string } -) => esClient.search(getRequestBody(params)); +): Promise => { + const { indexPattern, startDate, endDate } = params; + + const startDateMoment = dateMath.parse(startDate); + const endDateMoment = dateMath.parse(endDate, { roundUp: true }); + + if ( + !startDateMoment || + !endDateMoment || + !startDateMoment.isValid() || + !endDateMoment.isValid() + ) { + throw new Error('Invalid date format in startDate or endDate'); + } + + const startDateMillis = startDateMoment.valueOf(); + const endDateMillis = endDateMoment.valueOf(); + + const indices = (await esClient.cat.indices({ + index: indexPattern, + format: 'json', + h: 'index,creation.date', + })) as FetchAvailableCatIndicesResponseRequired; + + const filteredIndices = indices.filter((indexInfo) => { + const creationDate: string = indexInfo['creation.date'] ?? ''; + const creationDateMillis = parseInt(creationDate, 10); + + if (isNaN(creationDateMillis)) { + return false; + } + + return creationDateMillis >= startDateMillis && creationDateMillis <= endDateMillis; + }); + + return { + aggregations: { + index: { + buckets: filteredIndices.map((indexInfo) => ({ + key: indexInfo.index, + })), + }, + }, + }; +}; From 49444c1fc7af63b64b2bcfea9727d3028000b066 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:08:17 +0000 Subject: [PATCH 2/4] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json index ceb43169165b4..cf31d7461b509 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json +++ b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json @@ -26,6 +26,7 @@ "@kbn/core-elasticsearch-server-mocks", "@kbn/core-elasticsearch-server", "@kbn/core-security-common", + "@kbn/datemath", ], "exclude": [ "target/**/*", From b9e6d3a9039a755f3bddc5ab51c6f9b8427380a7 Mon Sep 17 00:00:00 2001 From: Karen Grigoryan Date: Mon, 21 Oct 2024 16:40:26 +0200 Subject: [PATCH 3/4] refactor: simplify, return string instead of aggregation --- .../lib/fetch_available_indices.test.ts | 24 +++---------------- .../server/lib/fetch_available_indices.ts | 22 ++--------------- .../server/lib/fetch_stats.ts | 14 +++++------ .../server/routes/get_index_stats.test.ts | 18 +++----------- .../server/routes/get_index_stats.ts | 7 ++---- 5 files changed, 17 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts index 1e71ba0b090ad..8a4515a9b1d4d 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts @@ -60,13 +60,7 @@ describe('fetchAvailableIndices', () => { endDate, }); - expect(result).toEqual({ - aggregations: { - index: { - buckets: [{ key: 'logs-2021.10.01' }, { key: 'logs-2021.10.05' }], - }, - }, - }); + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); expect(esClientMock.cat.indices).toHaveBeenCalledWith({ index: 'logs-*', @@ -97,13 +91,7 @@ describe('fetchAvailableIndices', () => { endDate, }); - expect(result).toEqual({ - aggregations: { - index: { - buckets: [], - }, - }, - }); + expect(result).toEqual([]); }); }); @@ -119,13 +107,7 @@ describe('fetchAvailableIndices', () => { endDate, }); - expect(result).toEqual({ - aggregations: { - index: { - buckets: [], - }, - }, - }); + expect(result).toEqual([]); }); }); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts index 7a1db0dea796f..94a2e1d278816 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts @@ -13,20 +13,10 @@ export type FetchAvailableCatIndicesResponseRequired = Array< Required> >; -export interface FetchAvailableIndicesResponse { - aggregations: { - index: { - buckets: Array<{ - key: string; - }>; - }; - }; -} - export const fetchAvailableIndices = async ( esClient: ElasticsearchClient, params: { indexPattern: string; startDate: string; endDate: string } -): Promise => { +): Promise => { const { indexPattern, startDate, endDate } = params; const startDateMoment = dateMath.parse(startDate); @@ -61,13 +51,5 @@ export const fetchAvailableIndices = async ( return creationDateMillis >= startDateMillis && creationDateMillis <= endDateMillis; }); - return { - aggregations: { - index: { - buckets: filteredIndices.map((indexInfo) => ({ - key: indexInfo.index, - })), - }, - }, - }; + return filteredIndices.map((indexInfo) => indexInfo.index); }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts index 536fd461c61c9..40fc59219342c 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts @@ -57,16 +57,16 @@ export const parseMeteringStats = (meteringStatsIndices: MeteringStatsIndex[]) = }, {}); export const pickAvailableMeteringStats = ( - indicesBuckets: Array<{ key: string }>, + indicesBuckets: string[], meteringStatsIndices: Record ) => - indicesBuckets.reduce((acc: Record, { key }: { key: string }) => { - if (meteringStatsIndices?.[key]) { - acc[key] = { - name: meteringStatsIndices?.[key].name, - num_docs: meteringStatsIndices?.[key].num_docs, + indicesBuckets.reduce((acc: Record, indexName: string) => { + if (meteringStatsIndices?.[indexName]) { + acc[indexName] = { + name: meteringStatsIndices?.[indexName].name, + num_docs: meteringStatsIndices?.[indexName].num_docs, size_in_bytes: null, // We don't have size_in_bytes intentionally when ILM is not available - data_stream: meteringStatsIndices?.[key].data_stream, + data_stream: meteringStatsIndices?.[indexName].data_stream, }; } return acc; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts index f3ff5ec256ad6..91996a4ab9f89 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts @@ -152,17 +152,7 @@ describe('getIndexStatsRoute route', () => { }, }; (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex); - (fetchAvailableIndices as jest.Mock).mockResolvedValue({ - aggregations: { - index: { - buckets: [ - { - key: 'my-index-000001', - }, - ], - }, - }, - }); + (fetchAvailableIndices as jest.Mock).mockResolvedValue(['my-index-000001']); const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(200); @@ -198,7 +188,7 @@ describe('getIndexStatsRoute route', () => { ); }); - test('returns an empty object when "availableIndices" indices are not available', async () => { + test('returns an empty object when "availableIndices" indices are empty', async () => { const request = requestMock.create({ method: 'get', path: GET_INDEX_STATS, @@ -214,9 +204,7 @@ describe('getIndexStatsRoute route', () => { const mockIndices = {}; (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex); - (fetchAvailableIndices as jest.Mock).mockResolvedValue({ - aggregations: undefined, - }); + (fetchAvailableIndices as jest.Mock).mockResolvedValue([]); const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(200); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index 665c178c62cdf..0f4fbb83f71e6 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -86,7 +86,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { endDate: decodedEndDate, }); - if (!availableIndices.aggregations?.index?.buckets) { + if (availableIndices.length === 0) { logger.warn( `No available indices found under pattern: ${decodedIndexName}, in the given date range: ${decodedStartDate} - ${decodedEndDate}` ); @@ -95,10 +95,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { }); } - const indices = pickAvailableMeteringStats( - availableIndices.aggregations.index.buckets, - meteringStatsIndices - ); + const indices = pickAvailableMeteringStats(availableIndices, meteringStatsIndices); return response.ok({ body: indices, From 64ea9110da29345159d1e64bda869c97ac461352 Mon Sep 17 00:00:00 2001 From: Karen Grigoryan Date: Fri, 25 Oct 2024 00:07:18 +0200 Subject: [PATCH 4/4] fix(fetch_available_indices): include indices with data within date range regardless of creation date Previously, `fetchAvailableIndices` only considered indices whose creation dates fell within the specified date range. This missed indices that were created outside the date range but contained data within it. This fix updates the function to also include indices that have data within the specified date range by performing an additional search aggregation. - Return back `getRequestBody` helper to build the search request for fetching indices with data in range. - Improve error handling for invalid date formats with more specific messages. - Update tests to reflect the new logic and error messages. --- .../server/helpers/get_available_indices.ts | 46 +++ .../lib/fetch_available_indices.test.ts | 329 ++++++++++++++++-- .../server/lib/fetch_available_indices.ts | 68 ++-- 3 files changed, 399 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts new file mode 100644 index 0000000000000..8f7fdead51547 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const getRequestBody = ({ + indexPattern, + startDate = 'now-7d/d', + endDate = 'now/d', +}: { + indexPattern: string; + startDate: string; + endDate: string; +}): SearchRequest => ({ + index: indexPattern, + aggs: { + index: { + terms: { + field: '_index', + }, + }, + }, + size: 0, + query: { + bool: { + must: [], + filter: [ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: startDate, + lte: endDate, + }, + }, + }, + ], + should: [], + must_not: [], + }, + }, +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts index 8a4515a9b1d4d..fa26fb68289a6 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts @@ -6,12 +6,23 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; +import moment from 'moment-timezone'; -import type { FetchAvailableCatIndicesResponseRequired } from './fetch_available_indices'; +import type { + FetchAvailableCatIndicesResponseRequired, + IndexSearchAggregationResponse, +} from './fetch_available_indices'; import { fetchAvailableIndices } from './fetch_available_indices'; function getEsClientMock() { return { + search: jest.fn().mockResolvedValue({ + aggregations: { + index: { + buckets: [], + }, + }, + }), cat: { indices: jest.fn().mockResolvedValue([]), }, @@ -19,23 +30,90 @@ function getEsClientMock() { cat: { indices: jest.Mock>; }; + search: jest.Mock>; }; } +// fixing timezone for both Date and moment +// so when tests are run in different timezones, the results are consistent +process.env.TZ = 'UTC'; +moment.tz.setDefault('UTC'); + const DAY_IN_MILLIS = 24 * 60 * 60 * 1000; -describe('fetchAvailableIndices', () => { - const startDate: string = '2021-10-01'; - const endDate: string = '2021-10-07'; +// We assume that the dates are in UTC, because es is using UTC +// It also diminishes difference date parsing by Date and moment constructors +// in different timezones, i.e. short ISO format '2021-10-01' is parsed as local +// date by moment and as UTC date by Date, whereas long ISO format '2021-10-01T00:00:00Z' +// is parsed as UTC date by both +const startDateString: string = '2021-10-01T00:00:00Z'; +const endDateString: string = '2021-10-07T00:00:00Z'; - const startDateMillis: number = new Date(startDate).getTime(); - const endDateMillis: number = new Date(endDate).getTime(); +const startDateMillis: number = new Date(startDateString).getTime(); +const endDateMillis: number = new Date(endDateString).getTime(); +describe('fetchAvailableIndices', () => { afterEach(() => { jest.clearAllMocks(); }); - describe('when indices are within the date range', () => { + it('aggregate search given index by startDate and endDate', async () => { + const esClientMock = getEsClientMock(); + + await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(esClientMock.search).toHaveBeenCalledWith({ + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: startDateString, + lte: endDateString, + }, + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + }, + index: 'logs-*', + size: 0, + aggs: { + index: { + terms: { + field: '_index', + }, + }, + }, + }); + }); + + it('should call esClient.cat.indices for given index', async () => { + const esClientMock = getEsClientMock(); + + await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(esClientMock.cat.indices).toHaveBeenCalledWith({ + index: 'logs-*', + format: 'json', + h: 'index,creation.date', + }); + }); + + describe('when indices are created within the date range', () => { it('returns indices within the date range', async () => { const esClientMock = getEsClientMock(); @@ -56,8 +134,8 @@ describe('fetchAvailableIndices', () => { const result = await fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, }); expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); @@ -87,8 +165,8 @@ describe('fetchAvailableIndices', () => { const result = await fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, }); expect(result).toEqual([]); @@ -96,21 +174,232 @@ describe('fetchAvailableIndices', () => { }); describe('when no indices match the index pattern', () => { - it('returns empty buckets', async () => { + it('returns empty list', async () => { const esClientMock = getEsClientMock(); esClientMock.cat.indices.mockResolvedValue([]); const result = await fetchAvailableIndices(esClientMock, { indexPattern: 'nonexistent-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual([]); + }); + }); + + describe('when indices have data in the date range', () => { + it('returns indices with data in the date range', async () => { + const esClientMock = getEsClientMock(); + + // esClient.cat.indices returns no indices + esClientMock.cat.indices.mockResolvedValue([]); + + // esClient.search returns indices with data in the date range + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [ + { key: 'logs-2021.10.02', doc_count: 100 }, + { key: 'logs-2021.10.03', doc_count: 150 }, + ], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.02', 'logs-2021.10.03']); + }); + + it('combines indices from both methods without duplicates', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.03', + 'creation.date': `${startDateMillis + 2 * DAY_IN_MILLIS}`, + }, + ]); + + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [ + { key: 'logs-2021.10.03', doc_count: 150 }, + { key: 'logs-2021.10.04', doc_count: 200 }, + ], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.03', 'logs-2021.10.04']); + }); + }); + + describe('edge cases for creation dates', () => { + it('includes indices with creation date exactly at startDate and endDate', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.07', + 'creation.date': `${endDateMillis}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.07']); + }); + }); + + describe('when esClient.search rejects', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.search.mockRejectedValue(new Error('Elasticsearch search error')); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }) + ).rejects.toThrow('Elasticsearch search error'); + }); + }); + + describe('when both esClient.cat.indices and esClient.search return empty', () => { + it('returns an empty list', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([]); + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, }); expect(result).toEqual([]); }); }); + describe('when indices are returned with both methods and have duplicates', () => { + it('does not duplicate indices in the result', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + ]); + + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [{ key: 'logs-2021.10.05', doc_count: 100 }], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.05']); + }); + }); + + describe('given keyword dates', () => { + describe('given 7 days range', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2021-10-07T00:00:00Z').getTime()); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('finds indices created within the date range', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + ]); + + const results = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'now-7d/d', + endDate: 'now/d', + }); + + expect(results).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); + }); + + it('finds indices with end date rounded up to the end of the day', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.06', + 'creation.date': `${new Date('2021-10-06T23:59:59Z').getTime()}`, + }, + ]); + + const results = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'now-7d/d', + endDate: 'now-1d/d', + }); + + expect(results).toEqual(['logs-2021.10.06']); + }); + }); + }); + describe('rejections', () => { beforeEach(() => { jest.spyOn(console, 'warn').mockImplementation(() => {}); @@ -127,8 +416,8 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, }) ).rejects.toThrow('Elasticsearch error'); }); @@ -142,9 +431,9 @@ describe('fetchAvailableIndices', () => { fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', startDate: 'invalid-date', - endDate, + endDate: endDateString, }) - ).rejects.toThrow('Invalid date format in startDate or endDate'); + ).rejects.toThrow('Invalid date format: invalid-date'); }); }); @@ -155,10 +444,10 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, + startDate: startDateString, endDate: 'invalid-date', }) - ).rejects.toThrow('Invalid date format in startDate or endDate'); + ).rejects.toThrow('Invalid date format: invalid-date'); }); }); }); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts index 94a2e1d278816..32311f28d636a 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts @@ -9,47 +9,67 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import type { CatIndicesIndicesRecord } from '@elastic/elasticsearch/lib/api/types'; import dateMath from '@kbn/datemath'; +import { getRequestBody } from '../helpers/get_available_indices'; + export type FetchAvailableCatIndicesResponseRequired = Array< Required> >; +type AggregateName = 'index'; +export interface IndexSearchAggregationResponse { + index: { + buckets: Array<{ key: string; doc_count: number }>; + }; +} + +const getParsedDateMs = (dateStr: string, roundUp = false) => { + const date = dateMath.parse(dateStr, roundUp ? { roundUp: true } : undefined); + if (!date?.isValid()) { + throw new Error(`Invalid date format: ${dateStr}`); + } + return date.valueOf(); +}; + export const fetchAvailableIndices = async ( esClient: ElasticsearchClient, params: { indexPattern: string; startDate: string; endDate: string } ): Promise => { const { indexPattern, startDate, endDate } = params; - const startDateMoment = dateMath.parse(startDate); - const endDateMoment = dateMath.parse(endDate, { roundUp: true }); - - if ( - !startDateMoment || - !endDateMoment || - !startDateMoment.isValid() || - !endDateMoment.isValid() - ) { - throw new Error('Invalid date format in startDate or endDate'); - } - - const startDateMillis = startDateMoment.valueOf(); - const endDateMillis = endDateMoment.valueOf(); + const startDateMs = getParsedDateMs(startDate); + const endDateMs = getParsedDateMs(endDate, true); - const indices = (await esClient.cat.indices({ + const indicesCats = (await esClient.cat.indices({ index: indexPattern, format: 'json', h: 'index,creation.date', })) as FetchAvailableCatIndicesResponseRequired; - const filteredIndices = indices.filter((indexInfo) => { - const creationDate: string = indexInfo['creation.date'] ?? ''; - const creationDateMillis = parseInt(creationDate, 10); + const indicesCatsInRange = indicesCats.filter((indexInfo) => { + const creationDateMs = parseInt(indexInfo['creation.date'], 10); + return creationDateMs >= startDateMs && creationDateMs <= endDateMs; + }); - if (isNaN(creationDateMillis)) { - return false; - } + const timeSeriesIndicesWithDataInRangeSearchResult = await esClient.search< + AggregateName, + IndexSearchAggregationResponse + >(getRequestBody(params)); - return creationDateMillis >= startDateMillis && creationDateMillis <= endDateMillis; - }); + const timeSeriesIndicesWithDataInRange = + timeSeriesIndicesWithDataInRangeSearchResult.aggregations?.index.buckets.map( + (bucket) => bucket.key + ) || []; + + // Combine indices from both sources removing duplicates + const resultingIndices = new Set(); + + for (const indicesCat of indicesCatsInRange) { + resultingIndices.add(indicesCat.index); + } + + for (const timeSeriesIndex of timeSeriesIndicesWithDataInRange) { + resultingIndices.add(timeSeriesIndex); + } - return filteredIndices.map((indexInfo) => indexInfo.index); + return Array.from(resultingIndices); };