diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts index 8d1302ebcec01..b45a3df2216c9 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.test.ts @@ -890,282 +890,6 @@ describe('es_query executor', () => { expect(mockSetLimitReached).toHaveBeenCalledTimes(1); expect(mockSetLimitReached).toHaveBeenCalledWith(false); }); - - it('should log messages for hits with out-of-range dates for search source', async () => { - const epoch = Date.now(); - const oneYear = 1000 * 60 * 60 * 24 * 365; - const dateEarly = new Date(epoch - oneYear).toISOString(); - const dateStart = new Date(epoch - 1000).toISOString(); - const dateMiddle = new Date(epoch - 500).toISOString(); - const dateEnd = new Date(epoch).toISOString(); - const dateLate = new Date(epoch + oneYear).toISOString(); - - function getTimeRange() { - return { dateStart, dateEnd }; - } - - mockFetchSearchSourceQuery.mockResolvedValueOnce({ - parsedResults: { - results: [ - { - group: 'all documents', - count: 3, - hits: [ - { _source: { '@timestamp': dateEarly, value: 1 } }, - { _source: { '@timestamp': dateMiddle, value: 2 } }, - { _source: { '@timestamp': dateLate, value: 3 } }, - ], - }, - ], - }, - truncated: false, - query: 'the query would go here', - }); - - const executorOptions: ExecutorOptions = { - ...defaultExecutorOptions, - getTimeRange, - params: { - ...defaultProps, - searchType: 'searchSource', - timeField: '@timestamp', - }, - }; - await executor(coreMock, executorOptions); - - const allLogCalls = loggerMock.collect(logger); - const messages: string[] = []; - for (const parms of allLogCalls.error) { - const message = parms.shift(); - messages.push(`${message}`); - } - - expect(messages).toEqual([ - `For rule 'test-rule-id', the hit with date '${dateEarly}' from field '@timestamp' is outside the query time range. Query: <\"the query would go here\">. Document: <{\"_source\":{\"@timestamp\":\"${dateEarly}\",\"value\":1}}>`, - `For rule 'test-rule-id', the hit with date '${dateLate}' from field '@timestamp' is outside the query time range. Query: <\"the query would go here\">. Document: <{\"_source\":{\"@timestamp\":\"${dateLate}\",\"value\":3}}>`, - ]); - expect(allLogCalls).toMatchInlineSnapshot(` - Object { - "debug": Array [], - "error": Array [ - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - ], - "fatal": Array [], - "info": Array [], - "log": Array [], - "trace": Array [], - "warn": Array [], - } - `); - }); - - it('should log messages for bad start / end dates for search source', async () => { - function getTimeRange() { - return { dateStart: 'x', dateEnd: 'y' }; - } - - mockFetchSearchSourceQuery.mockResolvedValueOnce({ - parsedResults: { - results: [ - { - group: 'all documents', - count: 1, - hits: [{ _source: { '@timestamp': new Date().toISOString() } }], - }, - ], - }, - truncated: false, - query: 'the query would go here', - }); - - const executorOptions: ExecutorOptions = { - ...defaultExecutorOptions, - getTimeRange, - params: { - ...defaultProps, - searchType: 'searchSource', - timeField: '@timestamp', - }, - }; - await executor(coreMock, executorOptions); - - const allLogCalls = loggerMock.collect(logger); - const messages: string[] = []; - for (const parms of allLogCalls.error) { - const message = parms.shift(); - messages.push(`${message}`); - } - - expect(messages).toEqual([ - `For rule 'test-rule-id', hits were returned with invalid time range start date 'x' from field '@timestamp' using query <"the query would go here">`, - `For rule 'test-rule-id', hits were returned with invalid time range end date 'y' from field '@timestamp' using query <"the query would go here">`, - ]); - expect(allLogCalls.error).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - ] - `); - }); - - it('should log messages for hits with out-of-range dates for query dsl', async () => { - const epoch = Date.now(); - const oneYear = 1000 * 60 * 60 * 24 * 365; - const dateEarly = new Date(epoch - oneYear).toISOString(); - const dateStart = new Date(epoch - 1000).toISOString(); - const dateMiddle = new Date(epoch - 500).toISOString(); - const dateEnd = new Date(epoch).toISOString(); - const dateLate = new Date(epoch + oneYear).toISOString(); - - function getTimeRange() { - return { dateStart, dateEnd }; - } - - mockFetchEsQuery.mockResolvedValueOnce({ - parsedResults: { - results: [ - { - group: 'all documents', - count: 3, - hits: [ - { _source: { '@timestamp': dateEarly, value: 1 } }, - { _source: { '@timestamp': dateMiddle, value: 2 } }, - { _source: { '@timestamp': dateLate, value: 3 } }, - ], - }, - ], - }, - truncated: false, - query: 'the query would go here', - }); - - const executorOptions: ExecutorOptions = { - ...defaultExecutorOptions, - getTimeRange, - params: { - ...defaultProps, - searchType: 'esQuery', - timeField: '@timestamp', - }, - }; - await executor(coreMock, executorOptions); - - const allLogCalls = loggerMock.collect(logger); - const messages: string[] = []; - for (const parms of allLogCalls.error) { - const message = parms.shift(); - messages.push(`${message}`); - } - - expect(messages).toEqual([ - `For rule 'test-rule-id', the hit with date '${dateEarly}' from field '@timestamp' is outside the query time range. Query: <\"the query would go here\">. Document: <{\"_source\":{\"@timestamp\":\"${dateEarly}\",\"value\":1}}>`, - `For rule 'test-rule-id', the hit with date '${dateLate}' from field '@timestamp' is outside the query time range. Query: <\"the query would go here\">. Document: <{\"_source\":{\"@timestamp\":\"${dateLate}\",\"value\":3}}>`, - ]); - expect(allLogCalls.error).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - ] - `); - }); - - it('should log messages for bad start / end dates for query dsl', async () => { - function getTimeRange() { - return { dateStart: 'x', dateEnd: 'y' }; - } - - mockFetchEsQuery.mockResolvedValueOnce({ - parsedResults: { - results: [ - { - group: 'all documents', - count: 1, - hits: [{ _source: { '@timestamp': new Date().toISOString() } }], - }, - ], - }, - truncated: false, - query: 'the query would go here', - }); - - const executorOptions: ExecutorOptions = { - ...defaultExecutorOptions, - getTimeRange, - params: { - ...defaultProps, - searchType: 'esQuery', - timeField: '@timestamp', - }, - }; - await executor(coreMock, executorOptions); - - const allLogCalls = loggerMock.collect(logger); - const messages: string[] = []; - for (const parms of allLogCalls.error) { - const message = parms.shift(); - messages.push(`${message}`); - } - - expect(messages).toEqual([ - `For rule 'test-rule-id', hits were returned with invalid time range start date 'x' from field '@timestamp' using query <"the query would go here">`, - `For rule 'test-rule-id', hits were returned with invalid time range end date 'y' from field '@timestamp' using query <"the query would go here">`, - ]); - expect(allLogCalls.error).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - Array [ - Object { - "tags": Array [ - "query-result-out-of-time-range", - ], - }, - ], - ] - `); - }); }); describe('tryToParseAsDate', () => { diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts index 0ada7928aeda3..43d40c866ee99 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/executor.ts @@ -7,7 +7,7 @@ import { sha256 } from 'js-sha256'; import { i18n } from '@kbn/i18n'; -import { CoreSetup, Logger } from '@kbn/core/server'; +import { CoreSetup } from '@kbn/core/server'; import { getEcsGroups } from '@kbn/observability-alerting-rule-utils'; import { isGroupAggregation, UngroupedGroupId } from '@kbn/triggers-actions-ui-plugin/common'; import { @@ -18,8 +18,6 @@ import { } from '@kbn/rule-data-utils'; import { AlertsClientError } from '@kbn/alerting-plugin/server'; -import { get } from 'lodash'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ComparatorFns } from '../../../common'; import { @@ -77,7 +75,7 @@ export async function executor(core: CoreSetup, options: ExecutorOptions>, - timeField: string | undefined, - dateStart: string, - dateEnd: string, - query: unknown -) { - if (!timeField) return; - - const epochStart = new Date(dateStart).getTime(); - const epochEnd = new Date(dateEnd).getTime(); - const messageMeta = { tags: ['query-result-out-of-time-range'] }; - - const messagePrefix = `For rule '${ruleId}'`; - const usingQuery = `using query <${JSON.stringify(query)}>`; - const hitsWereReturned = 'hits were returned with invalid time range'; - - let errors = 0; - if (isNaN(epochStart)) { - errors++; - logger.error( - `${messagePrefix}, ${hitsWereReturned} start date '${dateStart}' from field '${timeField}' ${usingQuery}`, - messageMeta - ); - } - - if (isNaN(epochEnd)) { - errors++; - logger.error( - `${messagePrefix}, ${hitsWereReturned} end date '${dateEnd}' from field '${timeField}' ${usingQuery}`, - messageMeta - ); - } - - if (errors > 0) return; - - const outsideTimeRange = 'outside the query time range'; - - for (const hit of hits) { - const dateVal = get(hit, [`_source`, timeField]); - const epochDate = getEpochDateFromString(dateVal); - - if (epochDate) { - if (epochDate < epochStart || epochDate > epochEnd) { - const message = `the hit with date '${dateVal}' from field '${timeField}' is ${outsideTimeRange}`; - const queryString = `Query: <${JSON.stringify(query)}>`; - const document = `Document: <${JSON.stringify(hit)}>`; - logger.error(`${messagePrefix}, ${message}. ${queryString}. ${document}`, messageMeta); - } - } - } -} - -function getEpochDateFromString(dateString: string): number | null { - let date: Date; - try { - date = new Date(dateString); - } catch (e) { - return null; - } - - const time = date.getTime(); - if (!isNaN(time)) return time; - - // if not a valid date string, try it as a stringified number - const dateNum = parseInt(dateString, 10); - if (isNaN(dateNum)) return null; - - const timeFromNumber = new Date(dateNum).getTime(); - if (isNaN(timeFromNumber)) return null; - - return timeFromNumber; -} - export function getValidTimefieldSort( sortValues: Array = [] ): undefined | string { diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_es_query.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_es_query.ts index 1e6c9a498c0ad..b42076952b87a 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_es_query.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_es_query.ts @@ -156,7 +156,6 @@ export async function fetchEsQuery({ sourceFieldsParams: params.sourceFields, }), link, - query: sortedQuery, index: params.index, }; } diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts index f017ee0f63aed..44f5a66f68c22 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts @@ -70,7 +70,6 @@ export async function fetchEsqlQuery({ return { link, - query, numMatches: Number(response.values.length), parsedResults: parseAggregationResults({ isCountAgg: true, diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts index 564e5cfeb58c6..7bc119dae7ed4 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts @@ -121,7 +121,6 @@ export async function fetchSearchSourceQuery({ termField: params.termField, }), index: [index.name], - query: searchRequestBody, }; }