diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 2ef72c22bbecf..1590a4f0fbb04 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -115,6 +115,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig export const sampleDocWithSortId = ( someUuid: string = sampleIdGuid, + sortIds: string[] = ['1234567891111', '2233447556677'], ip?: string | string[], destIp?: string | string[] ): SignalSourceHit => ({ @@ -139,7 +140,7 @@ export const sampleDocWithSortId = ( 'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'], 'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'], }, - sort: ['1234567891111'], + sort: sortIds, }); export const sampleDocNoSortId = ( @@ -630,7 +631,8 @@ export const repeatedSearchResultsWithSortId = ( pageSize: number, guids: string[], ips?: Array, - destIps?: Array + destIps?: Array, + sortIds?: string[] ): SignalSearchResponse => ({ took: 10, timed_out: false, @@ -646,6 +648,7 @@ export const repeatedSearchResultsWithSortId = ( hits: Array.from({ length: pageSize }).map((x, index) => ({ ...sampleDocWithSortId( guids[index], + sortIds, ips ? ips[index] : '127.0.0.1', destIps ? destIps[index] : '127.0.0.1' ), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index 4b74f865c6a53..3f4a17dc091ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -15,9 +15,8 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - searchAfterSortId: undefined, + searchAfterSortIds: undefined, timestampOverride: undefined, - excludeDocsWithTimestampOverride: false, }); expect(query).toEqual({ allow_no_indices: true, @@ -39,12 +38,19 @@ describe('create_signals', () => { bool: { filter: [ { - range: { - '@timestamp': { - gte: 'now-5m', - lte: 'today', - format: 'strict_date_optional_time', - }, + bool: { + minimum_should_match: 1, + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + ], }, }, ], @@ -73,16 +79,16 @@ describe('create_signals', () => { }, }); }); - test('if searchAfterSortId is an empty string it should not be included', () => { + + test('it builds a now-5m up to today filter with timestampOverride', () => { const query = buildEventsSearchQuery({ index: ['auditbeat-*'], from: 'now-5m', to: 'today', filter: {}, size: 100, - searchAfterSortId: '', - timestampOverride: undefined, - excludeDocsWithTimestampOverride: false, + searchAfterSortIds: undefined, + timestampOverride: 'event.ingested', }); expect(query).toEqual({ allow_no_indices: true, @@ -91,6 +97,10 @@ describe('create_signals', () => { ignore_unavailable: true, body: { docvalue_fields: [ + { + field: 'event.ingested', + format: 'strict_date_optional_time', + }, { field: '@timestamp', format: 'strict_date_optional_time', @@ -104,12 +114,43 @@ describe('create_signals', () => { bool: { filter: [ { - range: { - '@timestamp': { - gte: 'now-5m', - lte: 'today', - format: 'strict_date_optional_time', - }, + bool: { + should: [ + { + range: { + 'event.ingested': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + { + bool: { + must_not: { + exists: { + field: 'event.ingested', + }, + }, + }, + }, + ], + }, + }, + ], + minimum_should_match: 1, }, }, ], @@ -128,6 +169,12 @@ describe('create_signals', () => { }, ], sort: [ + { + 'event.ingested': { + order: 'asc', + unmapped_type: 'date', + }, + }, { '@timestamp': { order: 'asc', @@ -138,7 +185,8 @@ describe('create_signals', () => { }, }); }); - test('if searchAfterSortId is a valid sortId string', () => { + + test('if searchAfterSortIds is a valid sortId string', () => { const fakeSortId = '123456789012'; const query = buildEventsSearchQuery({ index: ['auditbeat-*'], @@ -146,9 +194,8 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - searchAfterSortId: fakeSortId, + searchAfterSortIds: [fakeSortId], timestampOverride: undefined, - excludeDocsWithTimestampOverride: false, }); expect(query).toEqual({ allow_no_indices: true, @@ -170,12 +217,19 @@ describe('create_signals', () => { bool: { filter: [ { - range: { - '@timestamp': { - gte: 'now-5m', - lte: 'today', - format: 'strict_date_optional_time', - }, + bool: { + minimum_should_match: 1, + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + ], }, }, ], @@ -205,7 +259,7 @@ describe('create_signals', () => { }, }); }); - test('if searchAfterSortId is a valid sortId number', () => { + test('if searchAfterSortIds is a valid sortId number', () => { const fakeSortIdNumber = 123456789012; const query = buildEventsSearchQuery({ index: ['auditbeat-*'], @@ -213,9 +267,8 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - searchAfterSortId: fakeSortIdNumber, + searchAfterSortIds: [fakeSortIdNumber], timestampOverride: undefined, - excludeDocsWithTimestampOverride: false, }); expect(query).toEqual({ allow_no_indices: true, @@ -237,12 +290,19 @@ describe('create_signals', () => { bool: { filter: [ { - range: { - '@timestamp': { - gte: 'now-5m', - lte: 'today', - format: 'strict_date_optional_time', - }, + bool: { + minimum_should_match: 1, + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + ], }, }, ], @@ -279,9 +339,8 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - searchAfterSortId: undefined, + searchAfterSortIds: undefined, timestampOverride: undefined, - excludeDocsWithTimestampOverride: false, }); expect(query).toEqual({ allow_no_indices: true, @@ -303,12 +362,19 @@ describe('create_signals', () => { bool: { filter: [ { - range: { - '@timestamp': { - gte: 'now-5m', - lte: 'today', - format: 'strict_date_optional_time', - }, + bool: { + minimum_should_match: 1, + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + ], }, }, ], @@ -352,9 +418,8 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - searchAfterSortId: undefined, + searchAfterSortIds: undefined, timestampOverride: undefined, - excludeDocsWithTimestampOverride: false, }); expect(query).toEqual({ allow_no_indices: true, @@ -371,12 +436,19 @@ describe('create_signals', () => { bool: { filter: [ { - range: { - '@timestamp': { - gte: 'now-5m', - lte: 'today', - format: 'strict_date_optional_time', - }, + bool: { + minimum_should_match: 1, + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'today', + format: 'strict_date_optional_time', + }, + }, + }, + ], }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index e086c862262c1..86fb51e4785ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -5,6 +5,8 @@ * 2.0. */ import type { estypes } from '@elastic/elasticsearch'; +import { SortResults } from '@elastic/elasticsearch/api/types'; +import { isEmpty } from 'lodash'; import { SortOrderOrUndefined, TimestampOverrideOrUndefined, @@ -18,9 +20,8 @@ interface BuildEventsSearchQuery { filter?: estypes.QueryContainer; size: number; sortOrder?: SortOrderOrUndefined; - searchAfterSortId: string | number | undefined; + searchAfterSortIds: SortResults | undefined; timestampOverride: TimestampOverrideOrUndefined; - excludeDocsWithTimestampOverride: boolean; } export const buildEventsSearchQuery = ({ @@ -30,10 +31,9 @@ export const buildEventsSearchQuery = ({ to, filter, size, - searchAfterSortId, + searchAfterSortIds, sortOrder, timestampOverride, - excludeDocsWithTimestampOverride, }: BuildEventsSearchQuery) => { const defaultTimeFields = ['@timestamp']; const timestamps = @@ -43,36 +43,62 @@ export const buildEventsSearchQuery = ({ format: 'strict_date_optional_time', })); - const sortField = - timestampOverride != null && !excludeDocsWithTimestampOverride - ? timestampOverride - : '@timestamp'; + const rangeFilter: estypes.QueryContainer[] = + timestampOverride != null + ? [ + { + range: { + [timestampOverride]: { + lte: to, + gte: from, + format: 'strict_date_optional_time', + }, + }, + }, + { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: to, + gte: from, + // @ts-expect-error + format: 'strict_date_optional_time', + }, + }, + }, + { + bool: { + must_not: { + exists: { + field: timestampOverride, + }, + }, + }, + }, + ], + }, + }, + ] + : [ + { + range: { + '@timestamp': { + lte: to, + gte: from, + format: 'strict_date_optional_time', + }, + }, + }, + ]; - const rangeFilter: estypes.QueryContainer[] = [ - { - range: { - [sortField]: { - lte: to, - gte: from, - format: 'strict_date_optional_time', - }, - }, - }, + const filterWithTime: estypes.QueryContainer[] = [ + // but tests contain undefined, so I suppose it's desired behaviour + // @ts-expect-error undefined in not assignable to QueryContainer + filter, + { bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } }, ]; - if (excludeDocsWithTimestampOverride) { - rangeFilter.push({ - bool: { - must_not: { - exists: { - field: timestampOverride, - }, - }, - }, - }); - } - // @ts-expect-error undefined in not assignable to QueryContainer - // but tests contain undefined, so I suppose it's desired behaviour - const filterWithTime: estypes.QueryContainer[] = [filter, { bool: { filter: rangeFilter } }]; const searchQuery = { allow_no_indices: true, @@ -99,22 +125,39 @@ export const buildEventsSearchQuery = ({ ], ...(aggregations ? { aggregations } : {}), sort: [ - { - [sortField]: { - order: sortOrder ?? 'asc', - unmapped_type: 'date', - }, - }, + ...(timestampOverride != null + ? [ + { + [timestampOverride]: { + order: sortOrder ?? 'asc', + unmapped_type: 'date', + }, + }, + { + '@timestamp': { + order: sortOrder ?? 'asc', + unmapped_type: 'date', + }, + }, + ] + : [ + { + '@timestamp': { + order: sortOrder ?? 'asc', + unmapped_type: 'date', + }, + }, + ]), ], }, }; - if (searchAfterSortId) { + if (searchAfterSortIds != null && !isEmpty(searchAfterSortIds)) { return { ...searchQuery, body: { ...searchQuery.body, - search_after: [searchAfterSortId], + search_after: searchAfterSortIds, }, }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts index aac0f47c28295..3fa5d1178b3ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts @@ -17,7 +17,7 @@ import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock' describe('filterEventsAgainstList', () => { let listClient = listMock.getListClient(); let exceptionItem = getExceptionListItemSchemaMock(); - let events = [sampleDocWithSortId('123', '1.1.1.1')]; + let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; beforeEach(() => { jest.clearAllMocks(); @@ -44,7 +44,7 @@ describe('filterEventsAgainstList', () => { }, ], }; - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; }); afterEach(() => { @@ -111,7 +111,7 @@ describe('filterEventsAgainstList', () => { }); test('it returns a single matched set as a JSON.stringify() set from the "events"', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; (exceptionItem.entries[0] as EntryList).field = 'source.ip'; const [{ matchedSet }] = await createFieldAndSetTuples({ listClient, @@ -124,7 +124,10 @@ describe('filterEventsAgainstList', () => { }); test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('456', undefined, '2.2.2.2'), + ]; (exceptionItem.entries[0] as EntryList).field = 'source.ip'; const [{ matchedSet }] = await createFieldAndSetTuples({ listClient, @@ -137,7 +140,7 @@ describe('filterEventsAgainstList', () => { }); test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => { - events = [sampleDocWithSortId('123', ['1.1.1.1', '2.2.2.2'])]; + events = [sampleDocWithSortId('123', undefined, ['1.1.1.1', '2.2.2.2'])]; (exceptionItem.entries[0] as EntryList).field = 'source.ip'; const [{ matchedSet }] = await createFieldAndSetTuples({ listClient, @@ -150,7 +153,10 @@ describe('filterEventsAgainstList', () => { }); test('it returns 2 fields when given two exception list items', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('456', undefined, '2.2.2.2'), + ]; exceptionItem.entries = [ { field: 'source.ip', @@ -182,7 +188,10 @@ describe('filterEventsAgainstList', () => { }); test('it returns two matched sets from two different events, one excluded, and one included', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('456', undefined, '2.2.2.2'), + ]; exceptionItem.entries = [ { field: 'source.ip', @@ -215,7 +224,10 @@ describe('filterEventsAgainstList', () => { }); test('it returns two fields from two different events', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('456', undefined, '2.2.2.2'), + ]; exceptionItem.entries = [ { field: 'source.ip', @@ -249,8 +261,8 @@ describe('filterEventsAgainstList', () => { test('it returns two matches from two different events', async () => { events = [ - sampleDocWithSortId('123', '1.1.1.1', '3.3.3.3'), - sampleDocWithSortId('456', '2.2.2.2', '5.5.5.5'), + sampleDocWithSortId('123', undefined, '1.1.1.1', '3.3.3.3'), + sampleDocWithSortId('456', undefined, '2.2.2.2', '5.5.5.5'), ]; exceptionItem.entries = [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts index aae4a7aae2b9e..743218f9ed940 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts @@ -14,7 +14,7 @@ import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock' describe('createSetToFilterAgainst', () => { let listClient = listMock.getListClient(); - let events = [sampleDocWithSortId('123', '1.1.1.1')]; + let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; beforeEach(() => { jest.clearAllMocks(); @@ -27,7 +27,7 @@ describe('createSetToFilterAgainst', () => { })) ) ); - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; }); afterEach(() => { @@ -49,7 +49,7 @@ describe('createSetToFilterAgainst', () => { }); test('it returns 1 field if the list returns a single item', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; const field = await createSetToFilterAgainst({ events, field: 'source.ip', @@ -68,7 +68,10 @@ describe('createSetToFilterAgainst', () => { }); test('it returns 2 fields if the list returns 2 items', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('123', undefined, '2.2.2.2'), + ]; const field = await createSetToFilterAgainst({ events, field: 'source.ip', @@ -87,7 +90,10 @@ describe('createSetToFilterAgainst', () => { }); test('it returns 0 fields if the field does not match up to a valid field within the event', async () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('123', undefined, '2.2.2.2'), + ]; const field = await createSetToFilterAgainst({ events, field: 'nonexistent.field', // field does not exist diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts index eb5c69e8abfe8..45a058b55d84b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts @@ -14,7 +14,7 @@ import { FieldSet } from './types'; describe('filterEvents', () => { let listClient = listMock.getListClient(); - let events = [sampleDocWithSortId('123', '1.1.1.1')]; + let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; beforeEach(() => { jest.clearAllMocks(); @@ -27,7 +27,7 @@ describe('filterEvents', () => { })) ) ); - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; }); afterEach(() => { @@ -35,7 +35,7 @@ describe('filterEvents', () => { }); test('it filters out the event if it is "included"', () => { - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; const fieldAndSetTuples: FieldSet[] = [ { field: 'source.ip', @@ -51,7 +51,7 @@ describe('filterEvents', () => { }); test('it does not filter out the event if it is "excluded"', () => { - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; const fieldAndSetTuples: FieldSet[] = [ { field: 'source.ip', @@ -67,7 +67,7 @@ describe('filterEvents', () => { }); test('it does NOT filter out the event if the field is not found', () => { - events = [sampleDocWithSortId('123', '1.1.1.1')]; + events = [sampleDocWithSortId('123', undefined, '1.1.1.1')]; const fieldAndSetTuples: FieldSet[] = [ { field: 'madeup.nonexistent', // field does not exist @@ -83,7 +83,10 @@ describe('filterEvents', () => { }); test('it does NOT filter out the event if it is in both an inclusion and exclusion list', () => { - events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')]; + events = [ + sampleDocWithSortId('123', undefined, '1.1.1.1'), + sampleDocWithSortId('123', undefined, '2.2.2.2'), + ]; const fieldAndSetTuples: FieldSet[] = [ { field: 'source.ip', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 9d9eefe844532..0c7723b6f4cc2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -426,6 +426,84 @@ describe('searchAfterAndBulkCreate', () => { expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); + test('should return success when empty string sortId present', async () => { + mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise({ + took: 100, + errors: false, + items: [ + { + create: { + _id: someGuids[0], + _index: 'myfakeindex', + status: 201, + }, + }, + { + create: { + _id: someGuids[1], + _index: 'myfakeindex', + status: 201, + }, + }, + { + create: { + _id: someGuids[2], + _index: 'myfakeindex', + status: 201, + }, + }, + { + create: { + _id: someGuids[3], + _index: 'myfakeindex', + status: 201, + }, + }, + ], + }) + ); + mockService.scopedClusterClient.asCurrentUser.search + .mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise( + repeatedSearchResultsWithSortId( + 4, + 4, + someGuids.slice(0, 3), + ['1.1.1.1', '2.2.2.2', '2.2.2.2', '2.2.2.2'], + // this is the case we are testing, if we receive an empty string for one of the sort ids. + ['', '2222222222222'] + ) + ) + ) + .mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise( + sampleDocSearchResultsNoSortIdNoHits() + ) + ); + + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + ruleSO, + tuples, + listClient, + exceptionsList: [], + services: mockService, + logger: mockLogger, + eventsTelemetry: undefined, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + pageSize: 1, + filter: undefined, + refresh: false, + buildRuleMessage, + }); + expect(success).toEqual(true); + expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); + expect(createdSignalsCount).toEqual(4); + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); + test('should return success when all search results are in the allowlist and no sortId present', async () => { const searchListItems: SearchListItemArraySchema = [ { ...getSearchListItemResponseMock(), value: ['1.1.1.1'] }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 0bc0039b54dba..08f8abe384d0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -5,9 +5,8 @@ * 2.0. */ -/* eslint-disable complexity */ - import { identity } from 'lodash'; +import { SortResults } from '@elastic/elasticsearch/api/types'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; import { filterEventsAgainstList } from './filters/filter_events_against_list'; @@ -19,6 +18,7 @@ import { createTotalHitsFromSearchResult, mergeReturns, mergeSearchResults, + getSafeSortIds, } from './utils'; import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types'; @@ -44,10 +44,8 @@ export const searchAfterAndBulkCreate = async ({ let toReturn = createSearchAfterReturnType(); // sortId tells us where to start our next consecutive search_after query - let sortId: string | undefined; + let sortIds: SortResults | undefined; let hasSortId = true; // default to true so we execute the search on initial run - let backupSortId: string | undefined; - let hasBackupSortId = ruleParams.timestampOverride ? true : false; // signalsCreatedCount keeps track of how many signals we have created, // to ensure we don't exceed maxSignals @@ -69,60 +67,12 @@ export const searchAfterAndBulkCreate = async ({ while (signalsCreatedCount < tuple.maxSignals) { try { let mergedSearchResults = createSearchResultReturnType(); - logger.debug(buildRuleMessage(`sortIds: ${sortId}`)); - - // if there is a timestampOverride param we always want to do a secondary search against @timestamp - if (ruleParams.timestampOverride != null && hasBackupSortId) { - // only execute search if we have something to sort on or if it is the first search - const { - searchResult: searchResultB, - searchDuration: searchDurationB, - searchErrors: searchErrorsB, - } = await singleSearchAfter({ - buildRuleMessage, - searchAfterSortId: backupSortId, - index: inputIndexPattern, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - // @ts-expect-error please, declare a type explicitly instead of unknown - filter, - pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), - timestampOverride: ruleParams.timestampOverride, - excludeDocsWithTimestampOverride: true, - }); - - // call this function setSortIdOrExit() - const lastSortId = searchResultB?.hits?.hits[searchResultB.hits.hits.length - 1]?.sort; - if (lastSortId != null && lastSortId.length !== 0) { - // @ts-expect-error @elastic/elasticsearch SortResults contains null not assignable to backupSortId - backupSortId = lastSortId[0]; - hasBackupSortId = true; - } else { - logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB')); - hasBackupSortId = false; - } - - mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]); - - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnTypeFromResponse({ - searchResult: mergedSearchResults, - timestampOverride: undefined, - }), - createSearchAfterReturnType({ - searchAfterTimes: [searchDurationB], - errors: searchErrorsB, - }), - ]); - } + logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); if (hasSortId) { const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ buildRuleMessage, - searchAfterSortId: sortId, + searchAfterSortIds: sortIds, index: inputIndexPattern, from: tuple.from.toISOString(), to: tuple.to.toISOString(), @@ -132,7 +82,6 @@ export const searchAfterAndBulkCreate = async ({ filter, pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, - excludeDocsWithTimestampOverride: false, }); mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); toReturn = mergeReturns([ @@ -147,10 +96,11 @@ export const searchAfterAndBulkCreate = async ({ }), ]); - const lastSortId = searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort; - if (lastSortId != null && lastSortId.length !== 0) { - // @ts-expect-error @elastic/elasticsearch SortResults contains null not assignable to sortId - sortId = lastSortId[0]; + const lastSortIds = getSafeSortIds( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort + ); + if (lastSortIds != null && lastSortIds.length !== 0) { + sortIds = lastSortIds; hasSortId = true; } else { hasSortId = false; @@ -236,7 +186,7 @@ export const searchAfterAndBulkCreate = async ({ sendAlertTelemetryEvents(logger, eventsTelemetry, filteredEvents, buildRuleMessage); } - if (!hasSortId && !hasBackupSortId) { + if (!hasSortId) { logger.debug(buildRuleMessage('ran out of sort ids to sort on')); break; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index cbffac6e7b455..a40459d312b9f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -34,7 +34,7 @@ describe('singleSearchAfter', () => { elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId()) ); const { searchResult } = await singleSearchAfter({ - searchAfterSortId: undefined, + searchAfterSortIds: undefined, index: [], from: 'now-360s', to: 'now', @@ -44,7 +44,6 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, buildRuleMessage, - excludeDocsWithTimestampOverride: false, }); expect(searchResult).toEqual(sampleDocSearchResultsNoSortId()); }); @@ -53,7 +52,7 @@ describe('singleSearchAfter', () => { elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId()) ); const { searchErrors } = await singleSearchAfter({ - searchAfterSortId: undefined, + searchAfterSortIds: undefined, index: [], from: 'now-360s', to: 'now', @@ -63,7 +62,6 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, buildRuleMessage, - excludeDocsWithTimestampOverride: false, }); expect(searchErrors).toEqual([]); }); @@ -104,7 +102,7 @@ describe('singleSearchAfter', () => { }) ); const { searchErrors } = await singleSearchAfter({ - searchAfterSortId: undefined, + searchAfterSortIds: undefined, index: [], from: 'now-360s', to: 'now', @@ -114,21 +112,20 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, buildRuleMessage, - excludeDocsWithTimestampOverride: false, }); expect(searchErrors).toEqual([ 'index: "index-123" reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', ]); }); test('if singleSearchAfter works with a given sort id', async () => { - const searchAfterSortId = '1234567891111'; + const searchAfterSortIds = ['1234567891111']; mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( elasticsearchClientMock.createSuccessTransportRequestPromise( sampleDocSearchResultsWithSortId() ) ); const { searchResult } = await singleSearchAfter({ - searchAfterSortId, + searchAfterSortIds, index: [], from: 'now-360s', to: 'now', @@ -138,18 +135,17 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, buildRuleMessage, - excludeDocsWithTimestampOverride: false, }); expect(searchResult).toEqual(sampleDocSearchResultsWithSortId()); }); test('if singleSearchAfter throws error', async () => { - const searchAfterSortId = '1234567891111'; + const searchAfterSortIds = ['1234567891111']; mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce( elasticsearchClientMock.createErrorTransportRequestPromise(new Error('Fake Error')) ); await expect( singleSearchAfter({ - searchAfterSortId, + searchAfterSortIds, index: [], from: 'now-360s', to: 'now', @@ -159,7 +155,6 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, buildRuleMessage, - excludeDocsWithTimestampOverride: false, }) ).rejects.toThrow('Fake Error'); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 9dcec1861f15d..57ed05bcb27cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -6,6 +6,7 @@ */ import type { estypes } from '@elastic/elasticsearch'; import { performance } from 'perf_hooks'; +import { SearchRequest, SortResults } from '@elastic/elasticsearch/api/types'; import { AlertInstanceContext, AlertInstanceState, @@ -23,7 +24,7 @@ import { interface SingleSearchAfterParams { aggregations?: Record; - searchAfterSortId: string | undefined; + searchAfterSortIds: SortResults | undefined; index: string[]; from: string; to: string; @@ -34,13 +35,12 @@ interface SingleSearchAfterParams { filter?: estypes.QueryContainer; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; - excludeDocsWithTimestampOverride: boolean; } // utilize search_after for paging results into bulk. export const singleSearchAfter = async ({ aggregations, - searchAfterSortId, + searchAfterSortIds, index, from, to, @@ -51,7 +51,6 @@ export const singleSearchAfter = async ({ sortOrder, timestampOverride, buildRuleMessage, - excludeDocsWithTimestampOverride, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; @@ -66,15 +65,16 @@ export const singleSearchAfter = async ({ filter, size: pageSize, sortOrder, - searchAfterSortId, + searchAfterSortIds, timestampOverride, - excludeDocsWithTimestampOverride, }); const start = performance.now(); const { body: nextSearchAfterResult, - } = await services.scopedClusterClient.asCurrentUser.search(searchAfterQuery); + } = await services.scopedClusterClient.asCurrentUser.search( + searchAfterQuery as SearchRequest + ); const end = performance.now(); const searchErrors = createErrorsFromShard({ errors: nextSearchAfterResult._shards.failures ?? [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_previous_threshold_signals.ts index 06e718b646ffa..1a2bfbf3a962d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_previous_threshold_signals.ts @@ -71,7 +71,7 @@ export const findPreviousThresholdSignals = async ({ }; return singleSearchAfter({ - searchAfterSortId: undefined, + searchAfterSortIds: undefined, timestampOverride, index: indexPattern, from, @@ -81,6 +81,5 @@ export const findPreviousThresholdSignals = async ({ filter, pageSize: 10000, // TODO: multiple pages? buildRuleMessage, - excludeDocsWithTimestampOverride: false, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts index 33ffa5b71a65c..986393d6d3454 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts @@ -141,6 +141,5 @@ export const findThresholdSignals = async ({ pageSize: 1, sortOrder: 'desc', buildRuleMessage, - excludeDocsWithTimestampOverride: false, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 54ed44956c8b3..bd37cf62c74b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -13,6 +13,7 @@ import type { estypes } from '@elastic/elasticsearch'; import { isEmpty, partition } from 'lodash'; import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport'; +import { SortResults } from '@elastic/elasticsearch/api/types'; import { TimestampOverrideOrUndefined, Privilege, @@ -846,3 +847,25 @@ export const isThreatParams = (params: RuleParams): params is ThreatRuleParams = params.type === 'threat_match'; export const isMachineLearningParams = (params: RuleParams): params is MachineLearningRuleParams => params.type === 'machine_learning'; + +/** + * Prevent javascript from returning Number.MAX_SAFE_INTEGER when Elasticsearch expects + * Java's Long.MAX_VALUE. This happens when sorting fields by date which are + * unmapped in the provided index + * + * Ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 + * + * return stringified Long.MAX_VALUE if we receive Number.MAX_SAFE_INTEGER + * @param sortIds SortResults | undefined + * @returns SortResults + */ +export const getSafeSortIds = (sortIds: SortResults | undefined) => { + return sortIds?.map((sortId) => { + // haven't determined when we would receive a null value for a sort id + // but in case we do, default to sending the stringified Java max_int + if (sortId == null || sortId === '' || sortId >= Number.MAX_SAFE_INTEGER) { + return '9223372036854775807'; + } + return sortId; + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index 6f437f7bcc8e5..ed758aa85bde9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -1322,5 +1322,118 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('auditbeat/hosts'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('auditbeat/hosts'); + }); + + /** + * This represents our worst case scenario where this field is not mapped on any index + * We want to check that our logic continues to function within the constraints of search after + * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields + * Javascript does not support numbers this large, but without passing in a number of this size + * The search_after will continue to return the same results and not iterate to the next set + * So to circumvent this limitation of javascript we return the stringified version of Java's + * Long.MAX_VALUE so that search_after does not enter into an infinite loop. + * + * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620 + */ + it('should generate 200 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['auditbeat-*']), + timestamp_override: 'event.fakeingested', + max_signals: 200, + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 200, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 200); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + + expect(signals.length).equal(200); + }); + }); + + /** + * Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override, + * then the documents will be queried and sorted using the timestamp override field. + * If no timestamp override field exists in the indices but one was provided to the rule, + * the rule's query will additionally search for events using the `@timestamp` field + */ + describe('Signals generated from events with timestamp override field', async () => { + beforeEach(async () => { + await deleteSignalsIndex(supertest); + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/timestamp_override_1'); + await esArchiver.load('security_solution/timestamp_override_2'); + await esArchiver.load('security_solution/timestamp_override_3'); + await esArchiver.load('security_solution/timestamp_override_4'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/timestamp_override_1'); + await esArchiver.unload('security_solution/timestamp_override_2'); + await esArchiver.unload('security_solution/timestamp_override_3'); + await esArchiver.unload('security_solution/timestamp_override_4'); + }); + + it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.ingested', + }; + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id], 3); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(3); + }); + + it('should generate 2 signals with @timestamp', async () => { + const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']); + + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + + it('should generate 2 signals when timestamp override does not exist', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfa*']), + timestamp_override: 'event.fakeingestfield', + }; + const { id } = await createRule(supertest, rule); + + await waitForRuleSuccessOrStatus(supertest, id, 'partial failure'); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsResponse = await getSignalsByIds(supertest, [id, id]); + const signals = signalsResponse.hits.hits.map((hit) => hit._source); + const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); + + expect(signalsOrderedByEventId.length).equal(2); + }); + }); }); }; diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/data.json.gz b/x-pack/test/functional/es_archives/security_solution/timestamp_override/data.json.gz index be351495c2f2e..a2c561471289f 100644 Binary files a/x-pack/test/functional/es_archives/security_solution/timestamp_override/data.json.gz and b/x-pack/test/functional/es_archives/security_solution/timestamp_override/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json index 28de7eeb2eb01..085ab34a3d58a 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json @@ -1,19 +1,19 @@ { - "type": "index", - "value": { - "index": "myfakeindex-1", - "mappings" : { - "properties" : { - "message" : { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword", - "ignore_above" : 256 - } - } - } + "type": "index", + "value": { + "index": "myfakeindex-1", + "mappings": { + "properties": { + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } } + } + } } -} \ No newline at end of file + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/data.json new file mode 100644 index 0000000000000..a07bf9fdd653b --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/data.json @@ -0,0 +1,10 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-1", + "source": { + "message": "hello world 1" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json new file mode 100644 index 0000000000000..085ab34a3d58a --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json @@ -0,0 +1,19 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-1", + "mappings": { + "properties": { + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/data.json new file mode 100644 index 0000000000000..24ba2aa42fb82 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/data.json @@ -0,0 +1,13 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-2", + "source": { + "message": "hello world 2", + "event": { + "ingested": "2020-12-16T15:16:18.570Z" + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json new file mode 100644 index 0000000000000..49a27a423cdaa --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json @@ -0,0 +1,26 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-2", + "mappings": { + "properties": { + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "ingested": { + "type": "date" + } + } + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/data.json new file mode 100644 index 0000000000000..56b0c8dff6eba --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/data.json @@ -0,0 +1,11 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-3", + "source": { + "message": "hello world 3", + "@timestamp": "2020-12-16T15:16:18.570Z" + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json new file mode 100644 index 0000000000000..736584386a705 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json @@ -0,0 +1,22 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-3", + "mappings": { + "properties": { + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "@timestamp": { + "type": "date" + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json new file mode 100644 index 0000000000000..ca7025b36154c --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json @@ -0,0 +1,14 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-4", + "source": { + "message": "hello world 4", + "@timestamp": "2020-12-16T15:16:18.570Z", + "event": { + "ingested": "2020-12-16T15:16:18.570Z" + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json new file mode 100644 index 0000000000000..ab4edc9f300e1 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json @@ -0,0 +1,29 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-4", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "ingested": { + "type": "date" + } + } + } + } + } + } +}