Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Security Solution] [Detections] Combine multiple timestamp searches into single request #96078

Merged
merged 13 commits into from
Apr 20, 2021
Merged

[Security Solution] [Detections] Combine multiple timestamp searches into single request #96078

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
7f557ae
merge multiple timestamp queries into one single search
dhurley14 Apr 1, 2021
bf50a37
fix types and unit tests
dhurley14 Apr 1, 2021
bd7252c
remove unused code for sending secondary search
dhurley14 Apr 2, 2021
b52d1db
removes unused excludeDocsWithTimestampOverride
dhurley14 Apr 2, 2021
1ff5050
adds integration tests to cover cases that should / should not genera…
dhurley14 Apr 14, 2021
a4de3be
adds integration test to ensure unmapped sort fields do not break sea…
dhurley14 Apr 15, 2021
3567909
Need to figure out why moving the tests around fixed them...
dhurley14 Apr 15, 2021
cc5269b
updates tests with new es archive data and fixes bug where exclusion …
dhurley14 Apr 15, 2021
8aad3e1
remove dead commented out code
dhurley14 Apr 15, 2021
d29c16f
fixes typo in test file, removes redundant delete signals call in int…
dhurley14 Apr 20, 2021
b3d932a
a unit test for checking if an empty string of a sort id is present w…
dhurley14 Apr 20, 2021
40b9b27
fix types
dhurley14 Apr 20, 2021
d155fed
removes isEmpty since it doesn't check for empty strings
dhurley14 Apr 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig

export const sampleDocWithSortId = (
someUuid: string = sampleIdGuid,
sortIds: string[] = ['1234567891111', '2233447556677'],
ip?: string | string[],
destIp?: string | string[]
): SignalSourceHit => ({
Expand All @@ -139,7 +140,7 @@ export const sampleDocWithSortId = (
'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'],
'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'],
},
sort: ['1234567891111'],
sort: sortIds,
});

export const sampleDocNoSortId = (
Expand Down Expand Up @@ -630,7 +631,8 @@ export const repeatedSearchResultsWithSortId = (
pageSize: number,
guids: string[],
ips?: Array<string | string[]>,
destIps?: Array<string | string[]>
destIps?: Array<string | string[]>,
sortIds?: string[]
): SignalSearchResponse => ({
took: 10,
timed_out: false,
Expand All @@ -646,6 +648,7 @@ export const repeatedSearchResultsWithSortId = (
hits: Array.from({ length: pageSize }).map((x, index) => ({
...sampleDocWithSortId(
guids[index],
sortIds,
ips ? ips[index] : '127.0.0.1',
destIps ? destIps[index] : '127.0.0.1'
),
Expand Down
176 changes: 124 additions & 52 deletions .../plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -39,12 +38,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -73,16 +79,16 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is an empty string it should not be included', () => {

test('it builds a now-5m up to today filter with timestampOverride', () => {
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: '',
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
searchAfterSortIds: undefined,
timestampOverride: 'event.ingested',
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -91,6 +97,10 @@ describe('create_signals', () => {
ignore_unavailable: true,
body: {
docvalue_fields: [
{
field: 'event.ingested',
format: 'strict_date_optional_time',
},
{
field: '@timestamp',
format: 'strict_date_optional_time',
Expand All @@ -104,12 +114,43 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
should: [
{
range: {
'event.ingested': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
bool: {
must_not: {
exists: {
field: 'event.ingested',
},
},
},
},
],
},
},
],
minimum_should_match: 1,
},
},
],
Expand All @@ -128,6 +169,12 @@ describe('create_signals', () => {
},
],
sort: [
{
'event.ingested': {
order: 'asc',
unmapped_type: 'date',
},
},
{
'@timestamp': {
order: 'asc',
Expand All @@ -138,17 +185,17 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is a valid sortId string', () => {

test('if searchAfterSortIds is a valid sortId string', () => {
const fakeSortId = '123456789012';
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: fakeSortId,
searchAfterSortIds: [fakeSortId],
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -170,12 +217,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -205,17 +259,16 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is a valid sortId number', () => {
test('if searchAfterSortIds is a valid sortId number', () => {
const fakeSortIdNumber = 123456789012;
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: fakeSortIdNumber,
searchAfterSortIds: [fakeSortIdNumber],
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -237,12 +290,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -279,9 +339,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -303,12 +362,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -352,9 +418,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -371,12 +436,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down
Loading