diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts new file mode 100644 index 0000000000000..8f7fdead51547 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const getRequestBody = ({ + indexPattern, + startDate = 'now-7d/d', + endDate = 'now/d', +}: { + indexPattern: string; + startDate: string; + endDate: string; +}): SearchRequest => ({ + index: indexPattern, + aggs: { + index: { + terms: { + field: '_index', + }, + }, + }, + size: 0, + query: { + bool: { + must: [], + filter: [ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: startDate, + lte: endDate, + }, + }, + }, + ], + should: [], + must_not: [], + }, + }, +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts index 8a4515a9b1d4d..fa26fb68289a6 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.test.ts @@ -6,12 +6,23 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; +import moment from 'moment-timezone'; -import type { FetchAvailableCatIndicesResponseRequired } from './fetch_available_indices'; +import type { + FetchAvailableCatIndicesResponseRequired, + IndexSearchAggregationResponse, +} from './fetch_available_indices'; import { fetchAvailableIndices } from './fetch_available_indices'; function getEsClientMock() { return { + search: jest.fn().mockResolvedValue({ + aggregations: { + index: { + buckets: [], + }, + }, + }), cat: { indices: jest.fn().mockResolvedValue([]), }, @@ -19,23 +30,90 @@ function getEsClientMock() { cat: { indices: jest.Mock>; }; + search: jest.Mock>; }; } +// fixing timezone for both Date and moment +// so when tests are run in different timezones, the results are consistent +process.env.TZ = 'UTC'; +moment.tz.setDefault('UTC'); + const DAY_IN_MILLIS = 24 * 60 * 60 * 1000; -describe('fetchAvailableIndices', () => { - const startDate: string = '2021-10-01'; - const endDate: string = '2021-10-07'; +// We assume that the dates are in UTC, because es is using UTC +// It also diminishes difference date parsing by Date and moment constructors +// in different timezones, i.e. short ISO format '2021-10-01' is parsed as local +// date by moment and as UTC date by Date, whereas long ISO format '2021-10-01T00:00:00Z' +// is parsed as UTC date by both +const startDateString: string = '2021-10-01T00:00:00Z'; +const endDateString: string = '2021-10-07T00:00:00Z'; - const startDateMillis: number = new Date(startDate).getTime(); - const endDateMillis: number = new Date(endDate).getTime(); +const startDateMillis: number = new Date(startDateString).getTime(); +const endDateMillis: number = new Date(endDateString).getTime(); +describe('fetchAvailableIndices', () => { afterEach(() => { jest.clearAllMocks(); }); - describe('when indices are within the date range', () => { + it('aggregate search given index by startDate and endDate', async () => { + const esClientMock = getEsClientMock(); + + await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(esClientMock.search).toHaveBeenCalledWith({ + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: startDateString, + lte: endDateString, + }, + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + }, + index: 'logs-*', + size: 0, + aggs: { + index: { + terms: { + field: '_index', + }, + }, + }, + }); + }); + + it('should call esClient.cat.indices for given index', async () => { + const esClientMock = getEsClientMock(); + + await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(esClientMock.cat.indices).toHaveBeenCalledWith({ + index: 'logs-*', + format: 'json', + h: 'index,creation.date', + }); + }); + + describe('when indices are created within the date range', () => { it('returns indices within the date range', async () => { const esClientMock = getEsClientMock(); @@ -56,8 +134,8 @@ describe('fetchAvailableIndices', () => { const result = await fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, }); expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); @@ -87,8 +165,8 @@ describe('fetchAvailableIndices', () => { const result = await fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, }); expect(result).toEqual([]); @@ -96,21 +174,232 @@ describe('fetchAvailableIndices', () => { }); describe('when no indices match the index pattern', () => { - it('returns empty buckets', async () => { + it('returns empty list', async () => { const esClientMock = getEsClientMock(); esClientMock.cat.indices.mockResolvedValue([]); const result = await fetchAvailableIndices(esClientMock, { indexPattern: 'nonexistent-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual([]); + }); + }); + + describe('when indices have data in the date range', () => { + it('returns indices with data in the date range', async () => { + const esClientMock = getEsClientMock(); + + // esClient.cat.indices returns no indices + esClientMock.cat.indices.mockResolvedValue([]); + + // esClient.search returns indices with data in the date range + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [ + { key: 'logs-2021.10.02', doc_count: 100 }, + { key: 'logs-2021.10.03', doc_count: 150 }, + ], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.02', 'logs-2021.10.03']); + }); + + it('combines indices from both methods without duplicates', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.03', + 'creation.date': `${startDateMillis + 2 * DAY_IN_MILLIS}`, + }, + ]); + + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [ + { key: 'logs-2021.10.03', doc_count: 150 }, + { key: 'logs-2021.10.04', doc_count: 200 }, + ], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.03', 'logs-2021.10.04']); + }); + }); + + describe('edge cases for creation dates', () => { + it('includes indices with creation date exactly at startDate and endDate', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.07', + 'creation.date': `${endDateMillis}`, + }, + ]); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.01', 'logs-2021.10.07']); + }); + }); + + describe('when esClient.search rejects', () => { + it('throws an error', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.search.mockRejectedValue(new Error('Elasticsearch search error')); + + await expect( + fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }) + ).rejects.toThrow('Elasticsearch search error'); + }); + }); + + describe('when both esClient.cat.indices and esClient.search return empty', () => { + it('returns an empty list', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([]); + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, }); expect(result).toEqual([]); }); }); + describe('when indices are returned with both methods and have duplicates', () => { + it('does not duplicate indices in the result', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + ]); + + esClientMock.search.mockResolvedValue({ + aggregations: { + index: { + buckets: [{ key: 'logs-2021.10.05', doc_count: 100 }], + }, + }, + }); + + const result = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: startDateString, + endDate: endDateString, + }); + + expect(result).toEqual(['logs-2021.10.05']); + }); + }); + + describe('given keyword dates', () => { + describe('given 7 days range', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2021-10-07T00:00:00Z').getTime()); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('finds indices created within the date range', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.01', + 'creation.date': `${startDateMillis}`, + }, + { + index: 'logs-2021.10.05', + 'creation.date': `${startDateMillis + 4 * DAY_IN_MILLIS}`, + }, + ]); + + const results = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'now-7d/d', + endDate: 'now/d', + }); + + expect(results).toEqual(['logs-2021.10.01', 'logs-2021.10.05']); + }); + + it('finds indices with end date rounded up to the end of the day', async () => { + const esClientMock = getEsClientMock(); + + esClientMock.cat.indices.mockResolvedValue([ + { + index: 'logs-2021.10.06', + 'creation.date': `${new Date('2021-10-06T23:59:59Z').getTime()}`, + }, + ]); + + const results = await fetchAvailableIndices(esClientMock, { + indexPattern: 'logs-*', + startDate: 'now-7d/d', + endDate: 'now-1d/d', + }); + + expect(results).toEqual(['logs-2021.10.06']); + }); + }); + }); + describe('rejections', () => { beforeEach(() => { jest.spyOn(console, 'warn').mockImplementation(() => {}); @@ -127,8 +416,8 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, - endDate, + startDate: startDateString, + endDate: endDateString, }) ).rejects.toThrow('Elasticsearch error'); }); @@ -142,9 +431,9 @@ describe('fetchAvailableIndices', () => { fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', startDate: 'invalid-date', - endDate, + endDate: endDateString, }) - ).rejects.toThrow('Invalid date format in startDate or endDate'); + ).rejects.toThrow('Invalid date format: invalid-date'); }); }); @@ -155,10 +444,10 @@ describe('fetchAvailableIndices', () => { await expect( fetchAvailableIndices(esClientMock, { indexPattern: 'logs-*', - startDate, + startDate: startDateString, endDate: 'invalid-date', }) - ).rejects.toThrow('Invalid date format in startDate or endDate'); + ).rejects.toThrow('Invalid date format: invalid-date'); }); }); }); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts index 94a2e1d278816..32311f28d636a 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_available_indices.ts @@ -9,47 +9,67 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import type { CatIndicesIndicesRecord } from '@elastic/elasticsearch/lib/api/types'; import dateMath from '@kbn/datemath'; +import { getRequestBody } from '../helpers/get_available_indices'; + export type FetchAvailableCatIndicesResponseRequired = Array< Required> >; +type AggregateName = 'index'; +export interface IndexSearchAggregationResponse { + index: { + buckets: Array<{ key: string; doc_count: number }>; + }; +} + +const getParsedDateMs = (dateStr: string, roundUp = false) => { + const date = dateMath.parse(dateStr, roundUp ? { roundUp: true } : undefined); + if (!date?.isValid()) { + throw new Error(`Invalid date format: ${dateStr}`); + } + return date.valueOf(); +}; + export const fetchAvailableIndices = async ( esClient: ElasticsearchClient, params: { indexPattern: string; startDate: string; endDate: string } ): Promise => { const { indexPattern, startDate, endDate } = params; - const startDateMoment = dateMath.parse(startDate); - const endDateMoment = dateMath.parse(endDate, { roundUp: true }); - - if ( - !startDateMoment || - !endDateMoment || - !startDateMoment.isValid() || - !endDateMoment.isValid() - ) { - throw new Error('Invalid date format in startDate or endDate'); - } - - const startDateMillis = startDateMoment.valueOf(); - const endDateMillis = endDateMoment.valueOf(); + const startDateMs = getParsedDateMs(startDate); + const endDateMs = getParsedDateMs(endDate, true); - const indices = (await esClient.cat.indices({ + const indicesCats = (await esClient.cat.indices({ index: indexPattern, format: 'json', h: 'index,creation.date', })) as FetchAvailableCatIndicesResponseRequired; - const filteredIndices = indices.filter((indexInfo) => { - const creationDate: string = indexInfo['creation.date'] ?? ''; - const creationDateMillis = parseInt(creationDate, 10); + const indicesCatsInRange = indicesCats.filter((indexInfo) => { + const creationDateMs = parseInt(indexInfo['creation.date'], 10); + return creationDateMs >= startDateMs && creationDateMs <= endDateMs; + }); - if (isNaN(creationDateMillis)) { - return false; - } + const timeSeriesIndicesWithDataInRangeSearchResult = await esClient.search< + AggregateName, + IndexSearchAggregationResponse + >(getRequestBody(params)); - return creationDateMillis >= startDateMillis && creationDateMillis <= endDateMillis; - }); + const timeSeriesIndicesWithDataInRange = + timeSeriesIndicesWithDataInRangeSearchResult.aggregations?.index.buckets.map( + (bucket) => bucket.key + ) || []; + + // Combine indices from both sources removing duplicates + const resultingIndices = new Set(); + + for (const indicesCat of indicesCatsInRange) { + resultingIndices.add(indicesCat.index); + } + + for (const timeSeriesIndex of timeSeriesIndicesWithDataInRange) { + resultingIndices.add(timeSeriesIndex); + } - return filteredIndices.map((indexInfo) => indexInfo.index); + return Array.from(resultingIndices); };