diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index 96a3be1f69..6d19bb312f 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -85,7 +85,7 @@ public PhysicalPlan visitLookup(LogicalLookup node, C context) { node.getAppendOnly(), node.getCopyFieldMap(), (a, b) -> { - throw new RuntimeException("not implemented by DefaultImplementor"); + throw new UnsupportedOperationException("Lookup not implemented by DefaultImplementor"); }); } diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index 9fb242a3bc..f2dfdfe661 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -143,7 +143,7 @@ public static LogicalPlan lookup( LogicalPlan input, String indexName, Map matchFieldMap, - boolean appendOnly, + Boolean appendOnly, Map copyFields) { return new LogicalLookup(input, indexName, matchFieldMap, appendOnly, copyFields); } diff --git a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java index 423163dc0e..930eb63a03 100644 --- a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java @@ -48,6 +48,7 @@ import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.executor.pagination.PlanSerializer; import org.opensearch.sql.expression.DSL; @@ -59,6 +60,7 @@ import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.expression.window.ranking.RowNumberFunction; import org.opensearch.sql.planner.logical.LogicalCloseCursor; +import org.opensearch.sql.planner.logical.LogicalLookup; import org.opensearch.sql.planner.logical.LogicalPaginate; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalPlanDSL; @@ -308,4 +310,19 @@ public void visitLookup_should_build_LookupOperator() { assertEquals(expectedPhysicalPlan, lookupOperator); } + + @Test + public void visitLookup_should_throw_unsupportedOperationException() { + LogicalLookup input = mock(LogicalLookup.class); + LogicalPlan dataSource = mock(LogicalPlan.class); + PhysicalPlan physicalSource = mock(PhysicalPlan.class); + when(dataSource.accept(implementor, null)).thenReturn(physicalSource); + when(input.getChild()).thenReturn(List.of(dataSource)); + PhysicalPlan lookupOperator = implementor.visitLookup(input, null); + when(physicalSource.next()).thenReturn(ExprValueUtils.tupleValue(Map.of("field", "value"))); + + var ex = assertThrows(UnsupportedOperationException.class, () -> lookupOperator.next()); + + assertEquals("Lookup not implemented by DefaultImplementor", ex.getMessage()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 8a0ad563a6..199af35553 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -709,7 +709,17 @@ public enum Index { TestsConstants.TEST_INDEX_NESTED_WITH_NULLS, "multi_nested", getNestedTypeIndexMapping(), - "src/test/resources/nested_with_nulls.json"); + "src/test/resources/nested_with_nulls.json"), + IOT_READINGS( + TestsConstants.TEST_INDEX_IOT_READINGS, + "iot_readings", + getMappingFile("iot_readings_index_mapping.json"), + "src/test/resources/iot_readings.json"), + IOT_SENSORS( + TestsConstants.TEST_INDEX_IOT_SENSORS, + "iot_sensors", + getMappingFile("iot_sensors_index_mapping.json"), + "src/test/resources/iot_sensors.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 29bc9813fa..6243c78fe4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -57,6 +57,8 @@ public class TestsConstants { public static final String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard"; public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; + public static final String TEST_INDEX_IOT_READINGS = TEST_INDEX + "_iot_readings"; + public static final String TEST_INDEX_IOT_SENSORS = TEST_INDEX + "_iot_sensors"; public static final String DATASOURCES = ".ql-datasources"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java index cb05285ea3..ade37b1241 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java @@ -5,8 +5,8 @@ package org.opensearch.sql.ppl; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_IOT_READINGS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_IOT_SENSORS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,47 +18,190 @@ public class LookupCommandIT extends PPLIntegTestCase { @Override public void init() throws IOException { - loadIndex(Index.BANK); - loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.IOT_READINGS); + loadIndex(Index.IOT_SENSORS); } @Test public void testLookup() throws IOException { - JSONObject result = executeQuery(String.format("source=%s | lookup %s male", TEST_INDEX_BANK)); - verifyDataRows(result, rows(true), rows(false)); + JSONObject result = + executeQuery( + String.format( + "source=%s | lookup %s did as device-id | sort @timestamp", + TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); + verifyDataRows( + result, + rows( + 28.1, + "2015-01-20 15:31:32.406431", + 255, + "temperature-basement", + "meter", + 255, + "VendorOne"), + rows( + 27.8, + "2016-01-20 15:31:33.509334", + 256, + "temperature-living-room", + "temperature meter", + 256, + "VendorTwo"), + rows( + 27.4, + "2017-01-20 15:31:35.732436", + 257, + "temperature-bedroom", + "camcorder", + 257, + "VendorThree"), + rows( + 28.5, + "2018-01-20 15:32:32.406431", + 255, + "temperature-basement", + "meter", + 255, + "VendorOne"), + rows( + 27.9, + "2019-01-20 15:32:33.509334", + 256, + "temperature-living-room", + "temperature meter", + 256, + "VendorTwo"), + rows( + 27.4, + "2020-01-20 15:32:35.732436", + 257, + "temperature-bedroom", + "camcorder", + 257, + "VendorThree")); } @Test - public void testConsecutiveDedup() throws IOException { + public void testLookupSelectedAttribute() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK)); - verifyDataRows(result, rows(true), rows(false), rows(true), rows(false)); + "source=%s | lookup %s did as device-id type, vendor | sort @timestamp", + TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); + verifyDataRows( + result, + rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter", "VendorOne"), + rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter", "VendorTwo"), + rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder", "VendorThree"), + rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter", "VendorOne"), + rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter", "VendorTwo"), + rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder", "VendorThree")); } @Test - public void testAllowMoreDuplicates() throws IOException { + public void testLookupRenameSelectedAttributes() throws IOException { JSONObject result = - executeQuery(String.format("source=%s | dedup 2 male | fields male", TEST_INDEX_BANK)); - verifyDataRows(result, rows(true), rows(true), rows(false), rows(false)); + executeQuery( + String.format( + "source=%s | lookup %s did as device-id did as dev_id, type as kind, vendor | sort" + + " @timestamp", + TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); + verifyDataRows( + result, + rows(28.1, "2015-01-20 15:31:32.406431", 255, 255, "meter", "VendorOne"), + rows(27.8, "2016-01-20 15:31:33.509334", 256, 256, "temperature meter", "VendorTwo"), + rows(27.4, "2017-01-20 15:31:35.732436", 257, 257, "camcorder", "VendorThree"), + rows(28.5, "2018-01-20 15:32:32.406431", 255, 255, "meter", "VendorOne"), + rows(27.9, "2019-01-20 15:32:33.509334", 256, 256, "temperature meter", "VendorTwo"), + rows(27.4, "2020-01-20 15:32:35.732436", 257, 257, "camcorder", "VendorThree")); + } + + @Test + public void testLookupSelectedMultipleAttributes() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | lookup %s did as device-id type | sort @timestamp", + TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); + verifyDataRows( + result, + rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter"), + rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter"), + rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder"), + rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter"), + rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter"), + rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder")); + } + + @Test + public void testLookupShouldAppendOnlyShouldBeFalseByDefault() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | rename temperature as vendor | lookup %s did as device-id | sort" + + " @timestamp", + TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); + verifyDataRows( + result, + rows("2015-01-20 15:31:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255), + rows( + "2016-01-20 15:31:33.509334", + 256, + "VendorTwo", + "temperature-living-room", + "temperature meter", + 256), + rows( + "2017-01-20 15:31:35.732436", + 257, + "VendorThree", + "temperature-bedroom", + "camcorder", + 257), + rows("2018-01-20 15:32:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255), + rows( + "2019-01-20 15:32:33.509334", + 256, + "VendorTwo", + "temperature-living-room", + "temperature meter", + 256), + rows( + "2020-01-20 15:32:35.732436", + 257, + "VendorThree", + "temperature-bedroom", + "camcorder", + 257)); } @Test - public void testKeepEmptyDedup() throws IOException { + public void testLookupWithAppendOnlyFalse() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | dedup balance keepempty=true | fields firstname, balance", - TEST_INDEX_BANK_WITH_NULL_VALUES)); + "source=%s | rename temperature as vendor | lookup %s did as device-id appendonly =" + + " true | sort @timestamp", + TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS)); verifyDataRows( result, - rows("Amber JOHnny", 39225), - rows("Hattie", null), - rows("Nanette", 32838), - rows("Dale", 4180), - rows("Elinor", null), - rows("Virginia", null), - rows("Dillard", 48086)); + rows("2015-01-20 15:31:32.406431", 255, 28.1, "temperature-basement", "meter", 255), + rows( + "2016-01-20 15:31:33.509334", + 256, + 27.8, + "temperature-living-room", + "temperature meter", + 256), + rows("2017-01-20 15:31:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257), + rows("2018-01-20 15:32:32.406431", 255, 28.5, "temperature-basement", "meter", 255), + rows( + "2019-01-20 15:32:33.509334", + 256, + 27.9, + "temperature-living-room", + "temperature meter", + 256), + rows("2020-01-20 15:32:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257)); } } diff --git a/integ-test/src/test/resources/indexDefinitions/iot_readings_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/iot_readings_index_mappings.json new file mode 100644 index 0000000000..a97c6c170e --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/iot_readings_index_mappings.json @@ -0,0 +1,24 @@ +{ + "mappings": { + "properties": { + "device-id": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "temperature": { + "type": "float" + }, + "timestamp": { + "type": "date" + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/indexDefinitions/iot_sensors_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/iot_sensors_index_mappings.json new file mode 100644 index 0000000000..e9682a390e --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/iot_sensors_index_mappings.json @@ -0,0 +1,36 @@ +{ + "mappings": { + "properties": { + "did": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "vendor": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/iot_readings.json b/integ-test/src/test/resources/iot_readings.json new file mode 100644 index 0000000000..be7e542e87 --- /dev/null +++ b/integ-test/src/test/resources/iot_readings.json @@ -0,0 +1,13 @@ +{ "index" : { "_id" : "1" } } +{ "device-id":255, "temperature":28.1, "@timestamp":"2015-01-20T15:31:32.406431+00:00" } +{ "index" : { "_id" : "2" } } +{ "device-id":256, "temperature":27.8, "@timestamp":"2016-01-20T15:31:33.509334+00:00" } +{ "index" : { "_id" : "3" } } +{ "device-id":257, "temperature":27.4, "@timestamp":"2017-01-20T15:31:35.732436+00:00" } +{ "index" : { "_id" : "4" } } +{ "device-id":255, "temperature":28.5, "@timestamp":"2018-01-20T15:32:32.406431+00:00" } +{ "index" : { "_id" : "5" } } +{ "device-id":256, "temperature":27.9, "@timestamp":"2019-01-20T15:32:33.509334+00:00" } +{ "index" : { "_id" : "6" } } +{ "device-id":257, "temperature":27.4, "@timestamp":"2020-01-20T15:32:35.732436+00:00" } +{ "index" : { "_id" : "7" } } diff --git a/integ-test/src/test/resources/iot_sensors.json b/integ-test/src/test/resources/iot_sensors.json new file mode 100644 index 0000000000..a36dd0aaa4 --- /dev/null +++ b/integ-test/src/test/resources/iot_sensors.json @@ -0,0 +1,6 @@ +{ "index" : { "_id" : "1" } } +{ "did" : 255, "name":"temperature-basement", "vendor":"VendorOne", "type":"meter"} +{ "index" : { "_id" : "2" } } +{ "did" : 256, "name":"temperature-living-room", "vendor":"VendorTwo", "type":"temperature meter" } +{ "index" : { "_id" : "3" } } +{ "did" : 257, "name":"temperature-bedroom", "vendor":"VendorThree", "type":"camcorder"}