Skip to content

Commit

Permalink
Added unit tests and csv-spec tests for lookup-join on the same index…
Browse files Browse the repository at this point in the history 
… twice
  • Loading branch information
@craigtaverner
craigtaverner committed Dec 13, 2024
1 parent 64f3523 commit a2bdc02
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 0 deletions.
45 changes: 45 additions & 0 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,51 @@ count:long | type:keyword | message:keyword
1 | Disconnected | Disconnected
;

lookupMessageFromIndexTwice
required_capability: join_lookup_v6

FROM sample_data
| LOOKUP JOIN message_types_lookup ON message
| RENAME message AS message1, type AS type1
| EVAL message = client_ip::keyword
| LOOKUP JOIN message_types_lookup ON message
| RENAME message AS message2, type AS type2
;
ignoreOrder:true

@timestamp:date | client_ip:ip | event_duration:long | message1:keyword | type1:keyword | message2:keyword | type2:keyword
2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success | 172.21.3.15 | null
2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error | 172.21.3.15 | null
2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error | 172.21.3.15 | null
2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error | 172.21.3.15 | null
2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Disconnected | 172.21.0.5 | null
2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | Success | 172.21.2.113 | null
2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success | 172.21.2.162 | null
;

lookupMessageFromIndexTwiceKeep
required_capability: join_lookup_v6

FROM sample_data
| LOOKUP JOIN message_types_lookup ON message
| RENAME message AS message1, type AS type1
| EVAL message = client_ip::keyword
| LOOKUP JOIN message_types_lookup ON message
| RENAME message AS message2, type AS type2
| KEEP @timestamp, client_ip, event_duration, message1, type1, message2, type2
;
ignoreOrder:true

@timestamp:date | client_ip:ip | event_duration:long | message1:keyword | type1:keyword | message2:keyword | type2:keyword
2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | Success | 172.21.3.15 | null
2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error | 172.21.3.15 | null
2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error | 172.21.3.15 | null
2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error | 172.21.3.15 | null
2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | Disconnected | 172.21.0.5 | null
2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | Success | 172.21.2.113 | null
2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | Success | 172.21.2.162 | null
;

###############################################
# Tests with clientips_lookup and message_types_lookup indexes
###############################################
Expand Down
77 changes: 77 additions & 0 deletions ...esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1459,6 +1459,83 @@ public void testMultiLookupJoinKeepAfter() {
);
}

public void testMultiLookupJoinSameIndex() {
assertFieldNames(
"""
FROM sample_data
| EVAL client_ip = client_ip::keyword
| LOOKUP JOIN clientips_lookup ON client_ip
| EVAL client_ip = message
| LOOKUP JOIN clientips_lookup ON client_ip""",
Set.of("*"), // With no KEEP we should keep all fields
Set.of() // since global field names are wildcarded, we don't need to wildcard any indices
);
}

public void testMultiLookupJoinSameIndexKeepBefore() {
assertFieldNames(
"""
FROM sample_data
| EVAL client_ip = client_ip::keyword
| KEEP @timestamp, client_ip, event_duration, message
| LOOKUP JOIN clientips_lookup ON client_ip
| EVAL client_ip = message
| LOOKUP JOIN clientips_lookup ON client_ip""",
Set.of("@timestamp", "client_ip", "event_duration", "message", "@timestamp.*", "client_ip.*", "event_duration.*", "message.*"),
Set.of("clientips_lookup") // Since there is no KEEP after the last JOIN, we need to wildcard the index
);
}

public void testMultiLookupJoinSameIndexKeepBetween() {
assertFieldNames(
"""
FROM sample_data
| EVAL client_ip = client_ip::keyword
| LOOKUP JOIN clientips_lookup ON client_ip
| KEEP @timestamp, client_ip, event_duration, message, env
| EVAL client_ip = message
| LOOKUP JOIN clientips_lookup ON client_ip""",
Set.of(
"@timestamp",
"client_ip",
"event_duration",
"message",
"env",
"@timestamp.*",
"client_ip.*",
"event_duration.*",
"message.*",
"env.*"
),
Set.of("clientips_lookup") // Since there is no KEEP after the last JOIN, we need to wildcard the index
);
}

public void testMultiLookupJoinSameIndexKeepAfter() {
assertFieldNames(
"""
FROM sample_data
| EVAL client_ip = client_ip::keyword
| LOOKUP JOIN clientips_lookup ON client_ip
| EVAL client_ip = message
| LOOKUP JOIN clientips_lookup ON client_ip
| KEEP @timestamp, client_ip, event_duration, message, env""",
Set.of(
"@timestamp",
"client_ip",
"event_duration",
"message",
"env",
"@timestamp.*",
"client_ip.*",
"event_duration.*",
"message.*",
"env.*"
),
Set.of() // Since the KEEP is after both JOINs, we can use the global field names
);
}

private Set<String> fieldNames(String query, Set<String> enrichPolicyMatchFields) {
EsqlSession.ListenerResult listenerResult = new EsqlSession.ListenerResult(null);
return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, listenerResult).fieldNames();
Expand Down

0 comments on commit a2bdc02

Please sign in to comment.