One unit test and integration tests for the lookup operator.

Signed-off-by: Lukasz Soszynski <[email protected]>
opensearch-project · Jul 2, 2024 · 5b65e61 · 5b65e61
1 parent d6a1fb4
commit 5b65e61
Show file tree

Hide file tree

Showing 10 changed files with 276 additions and 25 deletions.
diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java
@@ -85,7 +85,7 @@ public PhysicalPlan visitLookup(LogicalLookup node, C context) {
         node.getAppendOnly(),
         node.getCopyFieldMap(),
         (a, b) -> {
-          throw new RuntimeException("not implemented by DefaultImplementor");
+          throw new UnsupportedOperationException("Lookup not implemented by DefaultImplementor");
         });
   }
 

diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java
@@ -143,7 +143,7 @@ public static LogicalPlan lookup(
       LogicalPlan input,
       String indexName,
       Map<ReferenceExpression, ReferenceExpression> matchFieldMap,
-      boolean appendOnly,
+      Boolean appendOnly,
       Map<ReferenceExpression, ReferenceExpression> copyFields) {
     return new LogicalLookup(input, indexName, matchFieldMap, appendOnly, copyFields);
   }

diff --git a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java
@@ -48,6 +48,7 @@
 import org.opensearch.sql.ast.tree.RareTopN.CommandType;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.data.model.ExprBooleanValue;
+import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.executor.pagination.PlanSerializer;
 import org.opensearch.sql.expression.DSL;
@@ -59,6 +60,7 @@
 import org.opensearch.sql.expression.window.WindowDefinition;
 import org.opensearch.sql.expression.window.ranking.RowNumberFunction;
 import org.opensearch.sql.planner.logical.LogicalCloseCursor;
+import org.opensearch.sql.planner.logical.LogicalLookup;
 import org.opensearch.sql.planner.logical.LogicalPaginate;
 import org.opensearch.sql.planner.logical.LogicalPlan;
 import org.opensearch.sql.planner.logical.LogicalPlanDSL;
@@ -308,4 +310,19 @@ public void visitLookup_should_build_LookupOperator() {
 
     assertEquals(expectedPhysicalPlan, lookupOperator);
   }
+
+  @Test
+  public void visitLookup_should_throw_unsupportedOperationException() {
+    LogicalLookup input = mock(LogicalLookup.class);
+    LogicalPlan dataSource = mock(LogicalPlan.class);
+    PhysicalPlan physicalSource = mock(PhysicalPlan.class);
+    when(dataSource.accept(implementor, null)).thenReturn(physicalSource);
+    when(input.getChild()).thenReturn(List.of(dataSource));
+    PhysicalPlan lookupOperator = implementor.visitLookup(input, null);
+    when(physicalSource.next()).thenReturn(ExprValueUtils.tupleValue(Map.of("field", "value")));
+
+    var ex = assertThrows(UnsupportedOperationException.class, () -> lookupOperator.next());
+
+    assertEquals("Lookup not implemented by DefaultImplementor", ex.getMessage());
+  }
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
@@ -709,7 +709,17 @@ public enum Index {
         TestsConstants.TEST_INDEX_NESTED_WITH_NULLS,
         "multi_nested",
         getNestedTypeIndexMapping(),
-        "src/test/resources/nested_with_nulls.json");
+        "src/test/resources/nested_with_nulls.json"),
+    IOT_READINGS(
+        TestsConstants.TEST_INDEX_IOT_READINGS,
+        "iot_readings",
+        getMappingFile("iot_readings_index_mapping.json"),
+        "src/test/resources/iot_readings.json"),
+    IOT_SENSORS(
+        TestsConstants.TEST_INDEX_IOT_SENSORS,
+        "iot_sensors",
+        getMappingFile("iot_sensors_index_mapping.json"),
+        "src/test/resources/iot_sensors.json");
 
     private final String name;
     private final String type;

diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
@@ -57,6 +57,8 @@ public class TestsConstants {
   public static final String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard";
   public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested";
   public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
+  public static final String TEST_INDEX_IOT_READINGS = TEST_INDEX + "_iot_readings";
+  public static final String TEST_INDEX_IOT_SENSORS = TEST_INDEX + "_iot_sensors";
   public static final String DATASOURCES = ".ql-datasources";
 
   public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LookupCommandIT.java
@@ -5,8 +5,8 @@
 
 package org.opensearch.sql.ppl;
 
-import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
-import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_IOT_READINGS;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_IOT_SENSORS;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
 
@@ -18,47 +18,190 @@ public class LookupCommandIT extends PPLIntegTestCase {
 
   @Override
   public void init() throws IOException {
-    loadIndex(Index.BANK);
-    loadIndex(Index.BANK_WITH_NULL_VALUES);
+    loadIndex(Index.IOT_READINGS);
+    loadIndex(Index.IOT_SENSORS);
   }
 
   @Test
   public void testLookup() throws IOException {
-    JSONObject result = executeQuery(String.format("source=%s | lookup %s male", TEST_INDEX_BANK));
-    verifyDataRows(result, rows(true), rows(false));
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | lookup %s did as device-id | sort @timestamp",
+                TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
+    verifyDataRows(
+        result,
+        rows(
+            28.1,
+            "2015-01-20 15:31:32.406431",
+            255,
+            "temperature-basement",
+            "meter",
+            255,
+            "VendorOne"),
+        rows(
+            27.8,
+            "2016-01-20 15:31:33.509334",
+            256,
+            "temperature-living-room",
+            "temperature meter",
+            256,
+            "VendorTwo"),
+        rows(
+            27.4,
+            "2017-01-20 15:31:35.732436",
+            257,
+            "temperature-bedroom",
+            "camcorder",
+            257,
+            "VendorThree"),
+        rows(
+            28.5,
+            "2018-01-20 15:32:32.406431",
+            255,
+            "temperature-basement",
+            "meter",
+            255,
+            "VendorOne"),
+        rows(
+            27.9,
+            "2019-01-20 15:32:33.509334",
+            256,
+            "temperature-living-room",
+            "temperature meter",
+            256,
+            "VendorTwo"),
+        rows(
+            27.4,
+            "2020-01-20 15:32:35.732436",
+            257,
+            "temperature-bedroom",
+            "camcorder",
+            257,
+            "VendorThree"));
   }
 
   @Test
-  public void testConsecutiveDedup() throws IOException {
+  public void testLookupSelectedAttribute() throws IOException {
     JSONObject result =
         executeQuery(
             String.format(
-                "source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK));
-    verifyDataRows(result, rows(true), rows(false), rows(true), rows(false));
+                "source=%s | lookup %s did as device-id type, vendor | sort @timestamp",
+                TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
+    verifyDataRows(
+        result,
+        rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter", "VendorOne"),
+        rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter", "VendorTwo"),
+        rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder", "VendorThree"),
+        rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter", "VendorOne"),
+        rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter", "VendorTwo"),
+        rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder", "VendorThree"));
   }
 
   @Test
-  public void testAllowMoreDuplicates() throws IOException {
+  public void testLookupRenameSelectedAttributes() throws IOException {
     JSONObject result =
-        executeQuery(String.format("source=%s | dedup 2 male | fields male", TEST_INDEX_BANK));
-    verifyDataRows(result, rows(true), rows(true), rows(false), rows(false));
+        executeQuery(
+            String.format(
+                "source=%s | lookup %s did as device-id  did as dev_id, type as kind, vendor | sort"
+                    + " @timestamp",
+                TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
+    verifyDataRows(
+        result,
+        rows(28.1, "2015-01-20 15:31:32.406431", 255, 255, "meter", "VendorOne"),
+        rows(27.8, "2016-01-20 15:31:33.509334", 256, 256, "temperature meter", "VendorTwo"),
+        rows(27.4, "2017-01-20 15:31:35.732436", 257, 257, "camcorder", "VendorThree"),
+        rows(28.5, "2018-01-20 15:32:32.406431", 255, 255, "meter", "VendorOne"),
+        rows(27.9, "2019-01-20 15:32:33.509334", 256, 256, "temperature meter", "VendorTwo"),
+        rows(27.4, "2020-01-20 15:32:35.732436", 257, 257, "camcorder", "VendorThree"));
+  }
+
+  @Test
+  public void testLookupSelectedMultipleAttributes() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | lookup %s did as device-id type | sort @timestamp",
+                TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
+    verifyDataRows(
+        result,
+        rows(28.1, "2015-01-20 15:31:32.406431", 255, "meter"),
+        rows(27.8, "2016-01-20 15:31:33.509334", 256, "temperature meter"),
+        rows(27.4, "2017-01-20 15:31:35.732436", 257, "camcorder"),
+        rows(28.5, "2018-01-20 15:32:32.406431", 255, "meter"),
+        rows(27.9, "2019-01-20 15:32:33.509334", 256, "temperature meter"),
+        rows(27.4, "2020-01-20 15:32:35.732436", 257, "camcorder"));
+  }
+
+  @Test
+  public void testLookupShouldAppendOnlyShouldBeFalseByDefault() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | rename temperature as vendor | lookup %s did as device-id | sort"
+                    + " @timestamp",
+                TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
+    verifyDataRows(
+        result,
+        rows("2015-01-20 15:31:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255),
+        rows(
+            "2016-01-20 15:31:33.509334",
+            256,
+            "VendorTwo",
+            "temperature-living-room",
+            "temperature meter",
+            256),
+        rows(
+            "2017-01-20 15:31:35.732436",
+            257,
+            "VendorThree",
+            "temperature-bedroom",
+            "camcorder",
+            257),
+        rows("2018-01-20 15:32:32.406431", 255, "VendorOne", "temperature-basement", "meter", 255),
+        rows(
+            "2019-01-20 15:32:33.509334",
+            256,
+            "VendorTwo",
+            "temperature-living-room",
+            "temperature meter",
+            256),
+        rows(
+            "2020-01-20 15:32:35.732436",
+            257,
+            "VendorThree",
+            "temperature-bedroom",
+            "camcorder",
+            257));
   }
 
   @Test
-  public void testKeepEmptyDedup() throws IOException {
+  public void testLookupWithAppendOnlyFalse() throws IOException {
     JSONObject result =
         executeQuery(
             String.format(
-                "source=%s | dedup balance keepempty=true | fields firstname, balance",
-                TEST_INDEX_BANK_WITH_NULL_VALUES));
+                "source=%s | rename temperature as vendor | lookup %s did as device-id appendonly ="
+                    + " true | sort @timestamp",
+                TEST_INDEX_IOT_READINGS, TEST_INDEX_IOT_SENSORS));
     verifyDataRows(
         result,
-        rows("Amber JOHnny", 39225),
-        rows("Hattie", null),
-        rows("Nanette", 32838),
-        rows("Dale", 4180),
-        rows("Elinor", null),
-        rows("Virginia", null),
-        rows("Dillard", 48086));
+        rows("2015-01-20 15:31:32.406431", 255, 28.1, "temperature-basement", "meter", 255),
+        rows(
+            "2016-01-20 15:31:33.509334",
+            256,
+            27.8,
+            "temperature-living-room",
+            "temperature meter",
+            256),
+        rows("2017-01-20 15:31:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257),
+        rows("2018-01-20 15:32:32.406431", 255, 28.5, "temperature-basement", "meter", 255),
+        rows(
+            "2019-01-20 15:32:33.509334",
+            256,
+            27.9,
+            "temperature-living-room",
+            "temperature meter",
+            256),
+        rows("2020-01-20 15:32:35.732436", 257, 27.4, "temperature-bedroom", "camcorder", 257));
   }
 }
diff --git a/integ-test/src/test/resources/indexDefinitions/iot_readings_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/iot_readings_index_mappings.json
@@ -0,0 +1,24 @@
+{
+  "mappings": {
+    "properties": {
+      "device-id": {
+        "type": "long"
+      },
+      "name": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
+      },
+      "temperature": {
+        "type": "float"
+      },
+      "timestamp": {
+        "type": "date"
+      }
+    }
+  }
+}
diff --git a/integ-test/src/test/resources/indexDefinitions/iot_sensors_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/iot_sensors_index_mappings.json
@@ -0,0 +1,36 @@
+{
+  "mappings": {
+    "properties": {
+      "did": {
+        "type": "long"
+      },
+      "name": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
+      },
+      "type": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
+      },
+      "vendor": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/integ-test/src/test/resources/iot_readings.json b/integ-test/src/test/resources/iot_readings.json
@@ -0,0 +1,13 @@
+{ "index" : { "_id" : "1" } }
+{ "device-id":255, "temperature":28.1, "@timestamp":"2015-01-20T15:31:32.406431+00:00" }
+{ "index" : { "_id" : "2" } }
+{ "device-id":256, "temperature":27.8, "@timestamp":"2016-01-20T15:31:33.509334+00:00" }
+{ "index" : { "_id" : "3" } }
+{ "device-id":257, "temperature":27.4, "@timestamp":"2017-01-20T15:31:35.732436+00:00" }
+{ "index" : { "_id" : "4" } }
+{ "device-id":255, "temperature":28.5, "@timestamp":"2018-01-20T15:32:32.406431+00:00" }
+{ "index" : { "_id" : "5" } }
+{ "device-id":256, "temperature":27.9, "@timestamp":"2019-01-20T15:32:33.509334+00:00" }
+{ "index" : { "_id" : "6" } }
+{ "device-id":257, "temperature":27.4, "@timestamp":"2020-01-20T15:32:35.732436+00:00" }
+{ "index" : { "_id" : "7" } }
diff --git a/integ-test/src/test/resources/iot_sensors.json b/integ-test/src/test/resources/iot_sensors.json
@@ -0,0 +1,6 @@
+{ "index" : { "_id" : "1" } }
+{ "did" : 255, "name":"temperature-basement", "vendor":"VendorOne", "type":"meter"}
+{ "index" : { "_id" : "2" } }
+{ "did" : 256, "name":"temperature-living-room", "vendor":"VendorTwo", "type":"temperature meter" }
+{ "index" : { "_id" : "3" } }
+{ "did" : 257, "name":"temperature-bedroom", "vendor":"VendorThree", "type":"camcorder"}