From 611aa7e7a04dee84644f5131fd92821044dd1b2c Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 11 Oct 2023 17:51:05 -0700 Subject: [PATCH 01/11] add 2.11 release notes Signed-off-by: YANGDB --- .../opensearch-sql.release-notes-2.11.0.0.md | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.11.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.11.0.0.md b/release-notes/opensearch-sql.release-notes-2.11.0.0.md new file mode 100644 index 0000000000..f99a309297 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.11.0.0.md @@ -0,0 +1,55 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.10.0 + +### Features + +### Enhancements +* [Backport 2.x] Enable PPL lang and add datasource to async query API in https://github.com/opensearch-project/sql/pull/2195 +* [Backport 2.x] Refactor Flint Auth in https://github.com/opensearch-project/sql/pull/2201 +* [Backport 2.x] Add conf for spark structured streaming job in https://github.com/opensearch-project/sql/pull/2203 +* [Backport 2.x] Submit long running job only when auto_refresh = false in https://github.com/opensearch-project/sql/pull/2209 +* [Backport 2.x] Bug Fix, handle DESC TABLE response in https://github.com/opensearch-project/sql/pull/2213 +* [Backport 2.x] Drop Index Implementation in https://github.com/opensearch-project/sql/pull/2217 +* [Backport 2.x] Enable PPL Queries in https://github.com/opensearch-project/sql/pull/2223 +* [Backport 2.11] Read extra Spark submit parameters from cluster settings in https://github.com/opensearch-project/sql/pull/2236 +* [Backport 2.11] Spark Execution Engine Config Refactor in https://github.com/opensearch-project/sql/pull/2266 +* [Backport 2.11] Provide auth.type and auth.role_arn paramters in GET Datasource API response. in https://github.com/opensearch-project/sql/pull/2283 +* [Backport 2.x] Add support for `date_nanos` and tests. (#337) in https://github.com/opensearch-project/sql/pull/2020 +* [Backport 2.x] Applied formatting improvements to Antlr files based on spotless changes (#2017) by @MitchellGale in https://github.com/opensearch-project/sql/pull/2023 +* [Backport 2.x] Revert "Guarantee datasource read api is strong consistent read (#1815)" in https://github.com/opensearch-project/sql/pull/2031 +* [Backport 2.x] Add _primary preference only for segment replication enabled indices in https://github.com/opensearch-project/sql/pull/2045 +* [Backport 2.x] Changed allowlist config to denylist ip config for datasource uri hosts in https://github.com/opensearch-project/sql/pull/2058 + +### Bug Fixes +* [Backport 2.x] fix broken link for connectors doc in https://github.com/opensearch-project/sql/pull/2199 +* [Backport 2.x] Fix response codes returned by JSON formatting them in https://github.com/opensearch-project/sql/pull/2200 +* [Backport 2.x] Bug fix, datasource API should be case sensitive in https://github.com/opensearch-project/sql/pull/2202 +* [Backport 2.11] Minor fix in dropping covering index in https://github.com/opensearch-project/sql/pull/2240 +* [Backport 2.11] Fix Unit tests for FlintIndexReader in https://github.com/opensearch-project/sql/pull/2242 +* [Backport 2.11] Bug Fix , delete OpenSearch index when DROP INDEX in https://github.com/opensearch-project/sql/pull/2252 +* [Backport 2.11] Correctly Set query status in https://github.com/opensearch-project/sql/pull/2232 +* [Backport 2.x] Exclude generated files from spotless in https://github.com/opensearch-project/sql/pull/2024 +* [Backport 2.x] Fix mockito core conflict. in https://github.com/opensearch-project/sql/pull/2131 +* [Backport 2.x] Fix `ASCII` function and groom UT for text functions. (#301) in https://github.com/opensearch-project/sql/pull/2029 +* [Backport 2.x] Fixed response codes For Requests With security exception. in https://github.com/opensearch-project/sql/pull/2036 + +### Documentation +* [Backport 2.x] Datasource description in https://github.com/opensearch-project/sql/pull/2138 +* [Backport 2.11] Add documentation for S3GlueConnector. in https://github.com/opensearch-project/sql/pull/2234 + +### Infrastructure +* [Backport 2.x] bump aws-encryption-sdk-java to 1.71 in https://github.com/opensearch-project/sql/pull/2057 +* [Backport 2.x] Run IT tests with security plugin (#335) #1986 by @MitchellGale in https://github.com/opensearch-project/sql/pull/2022 + +### Refactoring +* [Backport 2.x] Merging Async Query APIs feature branch into main. in https://github.com/opensearch-project/sql/pull/2163 +* [Backport 2.x] Removed Domain Validation in https://github.com/opensearch-project/sql/pull/2136 +* [Backport 2.x] Check for existence of security plugin in https://github.com/opensearch-project/sql/pull/2069 +* [Backport 2.x] Always use snapshot version for security plugin download in https://github.com/opensearch-project/sql/pull/2061 +* [Backport 2.x] Add customized result index in data source etc in https://github.com/opensearch-project/sql/pull/2220 + +### Security +* [2.x] bump okhttp to 4.10.0 (#2043) by @joshuali925 in https://github.com/opensearch-project/sql/pull/2044 +* [2.x] bump okio to 3.4.0 by @joshuali925 in https://github.com/opensearch-project/sql/pull/2047 + +--- +**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...v.2.11.0.0 \ No newline at end of file From 2b4ab28399572c7cedb088b6f4be4f6180294081 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 11 Oct 2023 17:51:20 -0700 Subject: [PATCH 02/11] add 2.11 release notes Signed-off-by: YANGDB --- release-notes/opensearch-sql.release-notes-2.11.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/opensearch-sql.release-notes-2.11.0.0.md b/release-notes/opensearch-sql.release-notes-2.11.0.0.md index f99a309297..7434f4e95a 100644 --- a/release-notes/opensearch-sql.release-notes-2.11.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.11.0.0.md @@ -1,4 +1,4 @@ -Compatible with OpenSearch and OpenSearch Dashboards Version 2.10.0 +Compatible with OpenSearch and OpenSearch Dashboards Version 2.11.0 ### Features From 61f96c8e60f5a7c299b04ccdc052954482d6cdcf Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 12 Oct 2023 09:02:50 -0700 Subject: [PATCH 03/11] update notes Signed-off-by: YANGDB --- .../opensearch-sql.release-notes-2.11.0.0.md | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/release-notes/opensearch-sql.release-notes-2.11.0.0.md b/release-notes/opensearch-sql.release-notes-2.11.0.0.md index 7434f4e95a..a560d5c8dd 100644 --- a/release-notes/opensearch-sql.release-notes-2.11.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.11.0.0.md @@ -3,53 +3,53 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.11.0 ### Features ### Enhancements -* [Backport 2.x] Enable PPL lang and add datasource to async query API in https://github.com/opensearch-project/sql/pull/2195 -* [Backport 2.x] Refactor Flint Auth in https://github.com/opensearch-project/sql/pull/2201 -* [Backport 2.x] Add conf for spark structured streaming job in https://github.com/opensearch-project/sql/pull/2203 -* [Backport 2.x] Submit long running job only when auto_refresh = false in https://github.com/opensearch-project/sql/pull/2209 -* [Backport 2.x] Bug Fix, handle DESC TABLE response in https://github.com/opensearch-project/sql/pull/2213 -* [Backport 2.x] Drop Index Implementation in https://github.com/opensearch-project/sql/pull/2217 -* [Backport 2.x] Enable PPL Queries in https://github.com/opensearch-project/sql/pull/2223 -* [Backport 2.11] Read extra Spark submit parameters from cluster settings in https://github.com/opensearch-project/sql/pull/2236 -* [Backport 2.11] Spark Execution Engine Config Refactor in https://github.com/opensearch-project/sql/pull/2266 -* [Backport 2.11] Provide auth.type and auth.role_arn paramters in GET Datasource API response. in https://github.com/opensearch-project/sql/pull/2283 -* [Backport 2.x] Add support for `date_nanos` and tests. (#337) in https://github.com/opensearch-project/sql/pull/2020 -* [Backport 2.x] Applied formatting improvements to Antlr files based on spotless changes (#2017) by @MitchellGale in https://github.com/opensearch-project/sql/pull/2023 -* [Backport 2.x] Revert "Guarantee datasource read api is strong consistent read (#1815)" in https://github.com/opensearch-project/sql/pull/2031 -* [Backport 2.x] Add _primary preference only for segment replication enabled indices in https://github.com/opensearch-project/sql/pull/2045 -* [Backport 2.x] Changed allowlist config to denylist ip config for datasource uri hosts in https://github.com/opensearch-project/sql/pull/2058 +* Enable PPL lang and add datasource to async query API in https://github.com/opensearch-project/sql/pull/2195 +* Refactor Flint Auth in https://github.com/opensearch-project/sql/pull/2201 +* Add conf for spark structured streaming job in https://github.com/opensearch-project/sql/pull/2203 +* Submit long running job only when auto_refresh = false in https://github.com/opensearch-project/sql/pull/2209 +* Bug Fix, handle DESC TABLE response in https://github.com/opensearch-project/sql/pull/2213 +* Drop Index Implementation in https://github.com/opensearch-project/sql/pull/2217 +* Enable PPL Queries in https://github.com/opensearch-project/sql/pull/2223 +* Read extra Spark submit parameters from cluster settings in https://github.com/opensearch-project/sql/pull/2236 +* Spark Execution Engine Config Refactor in https://github.com/opensearch-project/sql/pull/2266 +* Provide auth.type and auth.role_arn paramters in GET Datasource API response. in https://github.com/opensearch-project/sql/pull/2283 +* Add support for `date_nanos` and tests. (#337) in https://github.com/opensearch-project/sql/pull/2020 +* Applied formatting improvements to Antlr files based on spotless changes (#2017) by @MitchellGale in https://github.com/opensearch-project/sql/pull/2023 +* Revert "Guarantee datasource read api is strong consistent read (#1815)" in https://github.com/opensearch-project/sql/pull/2031 +* Add _primary preference only for segment replication enabled indices in https://github.com/opensearch-project/sql/pull/2045 +* Changed allowlist config to denylist ip config for datasource uri hosts in https://github.com/opensearch-project/sql/pull/2058 ### Bug Fixes -* [Backport 2.x] fix broken link for connectors doc in https://github.com/opensearch-project/sql/pull/2199 -* [Backport 2.x] Fix response codes returned by JSON formatting them in https://github.com/opensearch-project/sql/pull/2200 -* [Backport 2.x] Bug fix, datasource API should be case sensitive in https://github.com/opensearch-project/sql/pull/2202 -* [Backport 2.11] Minor fix in dropping covering index in https://github.com/opensearch-project/sql/pull/2240 -* [Backport 2.11] Fix Unit tests for FlintIndexReader in https://github.com/opensearch-project/sql/pull/2242 -* [Backport 2.11] Bug Fix , delete OpenSearch index when DROP INDEX in https://github.com/opensearch-project/sql/pull/2252 -* [Backport 2.11] Correctly Set query status in https://github.com/opensearch-project/sql/pull/2232 -* [Backport 2.x] Exclude generated files from spotless in https://github.com/opensearch-project/sql/pull/2024 -* [Backport 2.x] Fix mockito core conflict. in https://github.com/opensearch-project/sql/pull/2131 -* [Backport 2.x] Fix `ASCII` function and groom UT for text functions. (#301) in https://github.com/opensearch-project/sql/pull/2029 -* [Backport 2.x] Fixed response codes For Requests With security exception. in https://github.com/opensearch-project/sql/pull/2036 +* fix broken link for connectors doc in https://github.com/opensearch-project/sql/pull/2199 +* Fix response codes returned by JSON formatting them in https://github.com/opensearch-project/sql/pull/2200 +* Bug fix, datasource API should be case sensitive in https://github.com/opensearch-project/sql/pull/2202 +* Minor fix in dropping covering index in https://github.com/opensearch-project/sql/pull/2240 +* Fix Unit tests for FlintIndexReader in https://github.com/opensearch-project/sql/pull/2242 +* Bug Fix , delete OpenSearch index when DROP INDEX in https://github.com/opensearch-project/sql/pull/2252 +* Correctly Set query status in https://github.com/opensearch-project/sql/pull/2232 +* Exclude generated files from spotless in https://github.com/opensearch-project/sql/pull/2024 +* Fix mockito core conflict. in https://github.com/opensearch-project/sql/pull/2131 +* Fix `ASCII` function and groom UT for text functions. (#301) in https://github.com/opensearch-project/sql/pull/2029 +* Fixed response codes For Requests With security exception. in https://github.com/opensearch-project/sql/pull/2036 ### Documentation -* [Backport 2.x] Datasource description in https://github.com/opensearch-project/sql/pull/2138 -* [Backport 2.11] Add documentation for S3GlueConnector. in https://github.com/opensearch-project/sql/pull/2234 +* Datasource description in https://github.com/opensearch-project/sql/pull/2138 +* Add documentation for S3GlueConnector. in https://github.com/opensearch-project/sql/pull/2234 ### Infrastructure -* [Backport 2.x] bump aws-encryption-sdk-java to 1.71 in https://github.com/opensearch-project/sql/pull/2057 -* [Backport 2.x] Run IT tests with security plugin (#335) #1986 by @MitchellGale in https://github.com/opensearch-project/sql/pull/2022 +* bump aws-encryption-sdk-java to 1.71 in https://github.com/opensearch-project/sql/pull/2057 +* Run IT tests with security plugin (#335) #1986 by @MitchellGale in https://github.com/opensearch-project/sql/pull/2022 ### Refactoring -* [Backport 2.x] Merging Async Query APIs feature branch into main. in https://github.com/opensearch-project/sql/pull/2163 -* [Backport 2.x] Removed Domain Validation in https://github.com/opensearch-project/sql/pull/2136 -* [Backport 2.x] Check for existence of security plugin in https://github.com/opensearch-project/sql/pull/2069 -* [Backport 2.x] Always use snapshot version for security plugin download in https://github.com/opensearch-project/sql/pull/2061 -* [Backport 2.x] Add customized result index in data source etc in https://github.com/opensearch-project/sql/pull/2220 +* Merging Async Query APIs feature branch into main. in https://github.com/opensearch-project/sql/pull/2163 +* Removed Domain Validation in https://github.com/opensearch-project/sql/pull/2136 +* Check for existence of security plugin in https://github.com/opensearch-project/sql/pull/2069 +* Always use snapshot version for security plugin download in https://github.com/opensearch-project/sql/pull/2061 +* Add customized result index in data source etc in https://github.com/opensearch-project/sql/pull/2220 ### Security -* [2.x] bump okhttp to 4.10.0 (#2043) by @joshuali925 in https://github.com/opensearch-project/sql/pull/2044 -* [2.x] bump okio to 3.4.0 by @joshuali925 in https://github.com/opensearch-project/sql/pull/2047 +* bump okhttp to 4.10.0 (#2043) by @joshuali925 in https://github.com/opensearch-project/sql/pull/2044 +* bump okio to 3.4.0 by @joshuali925 in https://github.com/opensearch-project/sql/pull/2047 --- **Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...v.2.11.0.0 \ No newline at end of file From f4158273930329a765e44c5fa1594561c44d635c Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 25 Oct 2023 23:22:52 -0700 Subject: [PATCH 04/11] append Struct / Array and Nested Json Object support for Flint related results Signed-off-by: YANGDB --- .../sql/data/model/ExprArrayValue.java | 64 +++++ .../opensearch/sql/data/model/ExprValue.java | 16 ++ .../src/main/antlr/FlintSparkSqlExtensions.g4 | 10 + spark/src/main/antlr/SparkSqlBase.g4 | 1 + ...DefaultSparkSqlFunctionResponseHandle.java | 121 ++++++++- .../spark/functions/response/JsonVisitor.java | 16 ++ ...ultSparkSqlFunctionResponseHandleTest.java | 245 ++++++++++++++++++ 7 files changed, 462 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java create mode 100644 spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java create mode 100644 spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java new file mode 100644 index 0000000000..d5f9626f73 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.data.model; + +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; + +import java.util.List; +import java.util.Objects; + +/** Expression array Value. */ +@RequiredArgsConstructor +public class ExprArrayValue extends AbstractExprValue { + private final List value; + + @Override + public boolean isArray() { + return true; + } + + @Override + public Object value() { + return value; + } + + @Override + public ExprType type() { + return ExprCoreType.ARRAY; + } + + @Override + public String stringValue() { + return value.stream().map(Object::toString).reduce("", String::concat); + } + + @Override + public List arrayValue() { + return value; + } + + @Override + public String toString() { + return String.format("\"%s\"", stringValue()); + } + + @Override + public int compare(ExprValue other) { + return stringValue().compareTo(other.stringValue()); + } + + @Override + public boolean equal(ExprValue other) { + return stringValue().equals(other.stringValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(stringValue()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java index 034ed22a75..c9429ded54 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java @@ -61,6 +61,16 @@ default boolean isDateTime() { return false; } + /** + * Is Array value. + * + * @return true: is a array value, otherwise false + */ + default boolean isArray() { + return false; + } + + /** Get the {@link BindingTuple}. */ default BindingTuple bindingTuples() { return BindingTuple.EMPTY; @@ -108,6 +118,12 @@ default String stringValue() { "invalid to get stringValue from value of type " + type()); } + /** Get array value. */ + default List arrayValue() { + throw new ExpressionEvaluationException( + "invalid to get arrayValue from value of type " + type()); + } + /** Get boolean value. */ default Boolean booleanValue() { throw new ExpressionEvaluationException( diff --git a/spark/src/main/antlr/FlintSparkSqlExtensions.g4 b/spark/src/main/antlr/FlintSparkSqlExtensions.g4 index f48c276e44..e44944fcff 100644 --- a/spark/src/main/antlr/FlintSparkSqlExtensions.g4 +++ b/spark/src/main/antlr/FlintSparkSqlExtensions.g4 @@ -31,6 +31,7 @@ createSkippingIndexStatement : CREATE SKIPPING INDEX (IF NOT EXISTS)? ON tableName LEFT_PAREN indexColTypeList RIGHT_PAREN + whereClause? (WITH LEFT_PAREN propertyList RIGHT_PAREN)? ; @@ -58,6 +59,7 @@ createCoveringIndexStatement : CREATE INDEX (IF NOT EXISTS)? indexName ON tableName LEFT_PAREN indexColumns=multipartIdentifierPropertyList RIGHT_PAREN + whereClause? (WITH LEFT_PAREN propertyList RIGHT_PAREN)? ; @@ -115,6 +117,14 @@ materializedViewQuery : .+? ; +whereClause + : WHERE filterCondition + ; + +filterCondition + : .+? + ; + indexColTypeList : indexColType (COMMA indexColType)* ; diff --git a/spark/src/main/antlr/SparkSqlBase.g4 b/spark/src/main/antlr/SparkSqlBase.g4 index 533d851ba6..597a1e5856 100644 --- a/spark/src/main/antlr/SparkSqlBase.g4 +++ b/spark/src/main/antlr/SparkSqlBase.g4 @@ -174,6 +174,7 @@ SHOW: 'SHOW'; TRUE: 'TRUE'; VIEW: 'VIEW'; VIEWS: 'VIEWS'; +WHERE: 'WHERE'; WITH: 'WITH'; diff --git a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java index 8fc417cd80..39fb850e59 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java +++ b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java @@ -9,10 +9,14 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Spliterator; +import java.util.stream.Collectors; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; +import org.opensearch.sql.data.model.ExprArrayValue; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -28,13 +32,58 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.ExecutionEngine; -/** Default implementation of SparkSqlFunctionResponseHandle. */ +import static java.util.Spliterators.spliteratorUnknownSize; +import static java.util.stream.StreamSupport.stream; +import static org.opensearch.sql.data.model.ExprValueUtils.fromObjectValue; + +/** Default implementation of SparkSqlFunctionResponseHandle. + * - expecting the following schema - + * { + * "data": { + * "applicationId": "some_spark_application_id", + * "schema": [ + * { + * "name": "column1", + * "type": "string" + * }, + * { + * "name": "column2", + * "type": "integer" + * }, + * { + * "name": "column3", + * "type": "boolean" + * } + * // ... more schema definitions + * ], + * "result": [ + * { + * "column1": "value1", + * "column2": 123, + * "column3": true + * }, + * { + * "column1": "value2", + * "column2": 456, + * "column3": false + * }, + * // ... more rows + * ] + * } + * } + */ public class DefaultSparkSqlFunctionResponseHandle implements SparkSqlFunctionResponseHandle { + public static final String DATA = "data"; + public static final String APPLICATION_ID = "applicationId"; + public static final String SCHEMA = "schema"; + public static final String RESULT = "result"; + public static final String COLUMN_NAME = "column_name"; + public static final String DATA_TYPE = "data_type"; private Iterator responseIterator; private ExecutionEngine.Schema schema; private static final Logger logger = LogManager.getLogger(DefaultSparkSqlFunctionResponseHandle.class); - + /** * Constructor. * @@ -47,19 +96,18 @@ public DefaultSparkSqlFunctionResponseHandle(JSONObject responseObject) { private void constructIteratorAndSchema(JSONObject responseObject) { List result = new ArrayList<>(); List columnList; - JSONObject items = responseObject.getJSONObject("data"); - logger.info("Spark Application ID: " + items.getString("applicationId")); - columnList = getColumnList(items.getJSONArray("schema")); - for (int i = 0; i < items.getJSONArray("result").length(); i++) { + JSONObject items = responseObject.getJSONObject(DATA); + logger.info("Spark Application ID: " + items.getString(APPLICATION_ID)); + columnList = getColumnList(items.getJSONArray(SCHEMA)); + for (int i = 0; i < items.getJSONArray(RESULT).length(); i++) { JSONObject row = - new JSONObject(items.getJSONArray("result").get(i).toString().replace("'", "\"")); + new JSONObject(items.getJSONArray(RESULT).get(i).toString().replace("'", "\"")); LinkedHashMap linkedHashMap = extractRow(row, columnList); result.add(new ExprTupleValue(linkedHashMap)); } this.schema = new ExecutionEngine.Schema(columnList); this.responseIterator = result.iterator(); } - private static LinkedHashMap extractRow( JSONObject row, List columnList) { LinkedHashMap linkedHashMap = new LinkedHashMap<>(); @@ -87,6 +135,10 @@ private static LinkedHashMap extractRow( column.getName(), new ExprTimestampValue(row.getString(column.getName()))); } else if (type == ExprCoreType.STRING) { linkedHashMap.put(column.getName(), new ExprStringValue(jsonString(row, column.getName()))); + } else if (type == ExprCoreType.STRUCT) { + linkedHashMap.put(column.getName(), exprStructBuilder(row, column.getName())); + } else if (type == ExprCoreType.ARRAY) { + linkedHashMap.put(column.getName(), exprArrayBuilder(row, column.getName())); } else { throw new RuntimeException("Result contains invalid data type"); } @@ -101,15 +153,19 @@ private List getColumnList(JSONArray schema) { JSONObject column = new JSONObject(schema.get(i).toString().replace("'", "\"")); columnList.add( new ExecutionEngine.Schema.Column( - column.get("column_name").toString(), - column.get("column_name").toString(), - getDataType(column.get("data_type").toString()))); + column.get(COLUMN_NAME).toString(), + column.get(COLUMN_NAME).toString(), + getDataType(column.get(DATA_TYPE).toString()))); } return columnList; } private ExprCoreType getDataType(String sparkDataType) { switch (sparkDataType) { + case "struct": + return ExprCoreType.STRUCT; + case "array": + return ExprCoreType.ARRAY; case "boolean": return ExprCoreType.BOOLEAN; case "long": @@ -141,6 +197,14 @@ private static String jsonString(JSONObject jsonObject, String key) { return jsonObject.has(key) ? jsonObject.getString(key) : ""; } + private static ExprArrayValue exprArrayBuilder(JSONObject jsonObject, String key) { + return (ExprArrayValue) new JsonVisitorImpl().visit(jsonObject.get(key)); + } + + private static ExprTupleValue exprStructBuilder(JSONObject jsonObject, String key) { + return (ExprTupleValue) new JsonVisitorImpl().visit(jsonObject.get(key)); + } + @Override public boolean hasNext() { return responseIterator.hasNext(); @@ -155,4 +219,39 @@ public ExprValue next() { public ExecutionEngine.Schema schema() { return schema; } + + /** + * visitor implementation for traversing the json object + */ + public static class JsonVisitorImpl implements JsonVisitor { + public ExprValue visit(Object obj) { + if (obj instanceof JSONObject) { + return visitObject((JSONObject) obj); + } else if (obj instanceof JSONArray) { + return visitArray((JSONArray) obj); + } else { + return visitPrimitive(obj); + } + } + + @Override + public ExprValue visitObject(JSONObject jsonObject) { + return fromObjectValue(jsonObject.keySet().stream().collect(Collectors.toMap( + key -> key, + key -> visit(jsonObject.get(key)) + ))); + } + + @Override + public ExprValue visitArray(JSONArray jsonArray) { + return new ExprArrayValue(stream(spliteratorUnknownSize(jsonArray.iterator(), Spliterator.ORDERED), false) + .map(this::visit) + .collect(Collectors.toList())); + } + + @Override + public ExprValue visitPrimitive(Object primitive) { + return fromObjectValue(primitive); + } + } } diff --git a/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java b/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java new file mode 100644 index 0000000000..f142d83e37 --- /dev/null +++ b/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java @@ -0,0 +1,16 @@ +package org.opensearch.sql.spark.functions.response; + +import org.json.JSONArray; +import org.json.JSONObject; + +/** + * visitor for the generic Json Object arriving from the flint's result payload + */ +public interface JsonVisitor { + + T visitObject(JSONObject jsonObject); + + T visitArray(JSONArray jsonArray); + + T visitPrimitive(Object primitive); +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java b/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java new file mode 100644 index 0000000000..1f11b3c66e --- /dev/null +++ b/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java @@ -0,0 +1,245 @@ +package org.opensearch.sql.spark.functions.response; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.executor.ExecutionEngine; + +import java.util.List; +import java.util.Map; + +import static java.lang.String.format; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.model.ExprTupleValue.fromExprValueMap; + +class DefaultSparkSqlFunctionResponseHandleTest { + + @Test + void testJsonObject() { + JSONObject jsonObject = new JSONObject("{" + + "\"data\": {" + + " \"applicationId\": \"some_spark_application_id\"," + + " \"schema\": [" + + " {\"column_name\": \"column1\", \"data_type\": \"string\"}," + + " {\"column_name\": \"column2\", \"data_type\": \"integer\"}," + + " {\"column_name\": \"column3\", \"data_type\": \"boolean\"}" + + " ]," + + " \"result\": [" + + " {\"column1\": \"value1\", \"column2\": 123, \"column3\": true}," + + " {\"column1\": \"value2\", \"column2\": 456, \"column3\": false}" + + " ]" + + "}" + + "}"); + DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals(new ExecutionEngine.Schema(List.of( + new ExecutionEngine.Schema.Column("column1", "column1", ExprCoreType.STRING), + new ExecutionEngine.Schema.Column("column2", "column2", ExprCoreType.INTEGER), + new ExecutionEngine.Schema.Column("column3", "column3", ExprCoreType.BOOLEAN))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + assertEquals(fromExprValueMap(Map.of("column1", new ExprStringValue("value1"), "column2", new ExprIntegerValue(123), "column3", ExprBooleanValue.of(true))), handle.next()); + assertEquals(fromExprValueMap(Map.of("column1", new ExprStringValue("value2"), "column2", new ExprIntegerValue(456), "column3", ExprBooleanValue.of(false))), handle.next()); + assertEquals(false, handle.hasNext()); + } + + @Test + void testJsonArrayObject() { + //region attributes + String data = " { 'attributes': [\n" + + " {\n" + + " 'key': 'telemetry.sdk.language',\n" + + " 'value': {\n" + + " 'stringValue': 'python'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'telemetry.sdk.name',\n" + + " 'value': {\n" + + " 'stringValue': 'opentelemetry'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'telemetry.sdk.version',\n" + + " 'value': {\n" + + " 'stringValue': '1.19.0'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'service.namespace',\n" + + " 'value': {\n" + + " 'stringValue': 'opentelemetry-demo'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'service.name',\n" + + " 'value': {\n" + + " 'stringValue': 'recommendationservice'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'telemetry.auto.version',\n" + + " 'value': {\n" + + " 'stringValue': '0.40b0'\n" + + " }\n" + + " }\n" + + " ]\n }"; + //endregion attributes + //region schema + String schema = "{'column_name':'attributes','data_type':'array'}"; + //endregion schema + + JSONObject jsonObject = new JSONObject(format("{" + "\"data\": "+ + " {\n" + + " \"applicationId\": \"00fd777k3k3ls20p\",\n" + + " \"schema\": [\n" + + " %s" + + " ],\n" + + " \"result\": [\n" + + " %s" + + " ],\n" + + " }\n" + + " }\n" + + " ]\n" + + " }" + + "}",schema,data)); + + DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals(new ExecutionEngine.Schema(List.of( + new ExecutionEngine.Schema.Column("attributes", "attributes", ExprCoreType.ARRAY))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); + assertEquals(1,tupleValue.tupleValue().size()); + assertEquals(true,tupleValue.tupleValue().get("attributes").isArray()); + assertEquals(6, tupleValue.tupleValue().get("attributes").arrayValue().size()); + assertEquals(false, handle.hasNext()); + } + + @Test + void testJsonNestedObject() { + //region resourceSpans + String data = "{\n" + + " 'resourceSpans': {\n" + + " 'scopeSpans': {\n" + + " 'spans': {\n" + + " 'key': 'rpc.system',\n" + + " 'value': {\n" + + " 'stringValue': 'grpc'\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + //endregion resourceSpans + //region schema + String schema = "{'column_name':'resourceSpans','data_type':'struct'}"; + //endregion schema + + JSONObject jsonObject = new JSONObject(format("{" + "\"data\": "+ + " {\n" + + " \"applicationId\": \"00fd777k3k3ls20p\",\n" + + " \"schema\": [\n" + + " %s" + + " ],\n" + + " \"result\": [\n" + + " %s" + + " ],\n" + + " }\n" + + " }\n" + + " ]\n" + + " }" + + "}",schema,data)); + + DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals(new ExecutionEngine.Schema(List.of( + new ExecutionEngine.Schema.Column("resourceSpans", "resourceSpans", ExprCoreType.STRUCT))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); + assertEquals(1,tupleValue.tupleValue().size()); + assertEquals("resourceSpans",tupleValue.tupleValue().keySet().iterator().next()); + assertEquals(1,tupleValue.keyValue("resourceSpans").tupleValue().size()); + assertEquals("scopeSpans",tupleValue.keyValue("resourceSpans").tupleValue().keySet().iterator().next()); + assertEquals("spans",tupleValue.keyValue("resourceSpans").keyValue("scopeSpans").tupleValue().keySet().iterator().next()); + assertEquals(2,tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").tupleValue().keySet().size()); + + assertEquals(false, handle.hasNext()); + + } + @Test + void testJsonNestedObjectArray() { + //region resourceSpans + String data = "{\n" + + " 'resourceSpans': \n" + + " {\n" + + " 'scopeSpans': \n" + + " {\n" + + " 'spans': \n" + + " [\n" + + " {\n" + + " 'attribute': {\n" + + " 'key': 'rpc.system',\n" + + " 'value': {\n" + + " 'stringValue': 'grpc'\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " 'attribute': {\n" + + " 'key': 'rpc.system',\n" + + " 'value': {\n" + + " 'stringValue': 'grpc'\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + //endregion resourceSpans + //region schema + String schema = "{'column_name':'resourceSpans','data_type':'struct'}"; + //endregion schema + + JSONObject jsonObject = new JSONObject(format("{" + "\"data\": "+ + " {\n" + + " \"applicationId\": \"00fd777k3k3ls20p\",\n" + + " \"schema\": [\n" + + " %s" + + " ],\n" + + " \"result\": [\n" + + " %s" + + " ],\n" + + " }\n" + + " }\n" + + " ]\n" + + " }" + + "}",schema,data)); + + DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals(new ExecutionEngine.Schema(List.of( + new ExecutionEngine.Schema.Column("resourceSpans", "resourceSpans", ExprCoreType.STRUCT))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); + assertEquals(1,tupleValue.tupleValue().size()); + assertEquals("resourceSpans",tupleValue.tupleValue().keySet().iterator().next()); + assertEquals(1,tupleValue.keyValue("resourceSpans").tupleValue().size()); + assertEquals("scopeSpans",tupleValue.keyValue("resourceSpans").tupleValue().keySet().iterator().next()); + assertEquals("spans",tupleValue.keyValue("resourceSpans").keyValue("scopeSpans").tupleValue().keySet().iterator().next()); + assertEquals(true,tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").isArray()); + assertEquals(2,tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").arrayValue().size()); + assertEquals("attribute",tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").arrayValue().get(0).tupleValue().keySet().iterator().next()); + assertEquals("attribute",tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").arrayValue().get(1).tupleValue().keySet().iterator().next()); + + assertEquals(false, handle.hasNext()); + } +} From b38062d139d7596352a6968631c6b089668a5336 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 25 Oct 2023 23:37:16 -0700 Subject: [PATCH 05/11] remove test which is no longer correct Signed-off-by: YANGDB --- .../SparkSqlFunctionTableScanOperatorTest.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/spark/src/test/java/org/opensearch/sql/spark/functions/SparkSqlFunctionTableScanOperatorTest.java b/spark/src/test/java/org/opensearch/sql/spark/functions/SparkSqlFunctionTableScanOperatorTest.java index 188cd695a3..78fa52c6a1 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/functions/SparkSqlFunctionTableScanOperatorTest.java +++ b/spark/src/test/java/org/opensearch/sql/spark/functions/SparkSqlFunctionTableScanOperatorTest.java @@ -141,23 +141,6 @@ void testQueryResponseAllTypes() { Assertions.assertFalse(sparkSqlFunctionTableScanOperator.hasNext()); } - @Test - @SneakyThrows - void testQueryResponseInvalidDataType() { - SparkQueryRequest sparkQueryRequest = new SparkQueryRequest(); - sparkQueryRequest.setSql(QUERY); - - SparkSqlFunctionTableScanOperator sparkSqlFunctionTableScanOperator = - new SparkSqlFunctionTableScanOperator(sparkClient, sparkQueryRequest); - - when(sparkClient.sql(any())).thenReturn(new JSONObject(getJson("invalid_data_type.json"))); - - RuntimeException exception = - Assertions.assertThrows( - RuntimeException.class, () -> sparkSqlFunctionTableScanOperator.open()); - Assertions.assertEquals("Result contains invalid data type", exception.getMessage()); - } - @Test @SneakyThrows void testQuerySchema() { From 29dd64ba51e1e72a1b31c5f5c5ca709d9c1c0b0d Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 25 Oct 2023 23:43:16 -0700 Subject: [PATCH 06/11] remove redundant comment Signed-off-by: YANGDB --- ...DefaultSparkSqlFunctionResponseHandle.java | 38 ++----------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java index 39fb850e59..b2c28c8061 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java +++ b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java @@ -36,41 +36,9 @@ import static java.util.stream.StreamSupport.stream; import static org.opensearch.sql.data.model.ExprValueUtils.fromObjectValue; -/** Default implementation of SparkSqlFunctionResponseHandle. - * - expecting the following schema - - * { - * "data": { - * "applicationId": "some_spark_application_id", - * "schema": [ - * { - * "name": "column1", - * "type": "string" - * }, - * { - * "name": "column2", - * "type": "integer" - * }, - * { - * "name": "column3", - * "type": "boolean" - * } - * // ... more schema definitions - * ], - * "result": [ - * { - * "column1": "value1", - * "column2": 123, - * "column3": true - * }, - * { - * "column1": "value2", - * "column2": 456, - * "column3": false - * }, - * // ... more rows - * ] - * } - * } +/** + * Default implementation of SparkSqlFunctionResponseHandle. + * */ public class DefaultSparkSqlFunctionResponseHandle implements SparkSqlFunctionResponseHandle { public static final String DATA = "data"; From e3f13c5a56a163101d5ed71ea62cbaa8e1daba78 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 26 Oct 2023 10:27:32 -0700 Subject: [PATCH 07/11] update spotless style issues Signed-off-by: YANGDB --- .../sql/data/model/ExprArrayValue.java | 5 +- .../opensearch/sql/data/model/ExprValue.java | 1 - ...DefaultSparkSqlFunctionResponseHandle.java | 31 +- .../spark/functions/response/JsonVisitor.java | 10 +- ...ultSparkSqlFunctionResponseHandleTest.java | 563 +++++++++++------- 5 files changed, 364 insertions(+), 246 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java index d5f9626f73..02ae1d16cf 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java @@ -5,13 +5,12 @@ package org.opensearch.sql.data.model; +import java.util.List; +import java.util.Objects; import lombok.RequiredArgsConstructor; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; -import java.util.List; -import java.util.Objects; - /** Expression array Value. */ @RequiredArgsConstructor public class ExprArrayValue extends AbstractExprValue { diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java index c9429ded54..b5c89ce6db 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java @@ -70,7 +70,6 @@ default boolean isArray() { return false; } - /** Get the {@link BindingTuple}. */ default BindingTuple bindingTuples() { return BindingTuple.EMPTY; diff --git a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java index b2c28c8061..faef49c071 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java +++ b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java @@ -5,13 +5,16 @@ package org.opensearch.sql.spark.functions.response; +import static java.util.Spliterators.spliteratorUnknownSize; +import static java.util.stream.StreamSupport.stream; +import static org.opensearch.sql.data.model.ExprValueUtils.fromObjectValue; + import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Spliterator; import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONArray; @@ -32,14 +35,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.ExecutionEngine; -import static java.util.Spliterators.spliteratorUnknownSize; -import static java.util.stream.StreamSupport.stream; -import static org.opensearch.sql.data.model.ExprValueUtils.fromObjectValue; - -/** - * Default implementation of SparkSqlFunctionResponseHandle. - * - */ +/** Default implementation of SparkSqlFunctionResponseHandle. */ public class DefaultSparkSqlFunctionResponseHandle implements SparkSqlFunctionResponseHandle { public static final String DATA = "data"; public static final String APPLICATION_ID = "applicationId"; @@ -51,7 +47,7 @@ public class DefaultSparkSqlFunctionResponseHandle implements SparkSqlFunctionRe private ExecutionEngine.Schema schema; private static final Logger logger = LogManager.getLogger(DefaultSparkSqlFunctionResponseHandle.class); - + /** * Constructor. * @@ -76,6 +72,7 @@ private void constructIteratorAndSchema(JSONObject responseObject) { this.schema = new ExecutionEngine.Schema(columnList); this.responseIterator = result.iterator(); } + private static LinkedHashMap extractRow( JSONObject row, List columnList) { LinkedHashMap linkedHashMap = new LinkedHashMap<>(); @@ -188,9 +185,7 @@ public ExecutionEngine.Schema schema() { return schema; } - /** - * visitor implementation for traversing the json object - */ + /** visitor implementation for traversing the json object */ public static class JsonVisitorImpl implements JsonVisitor { public ExprValue visit(Object obj) { if (obj instanceof JSONObject) { @@ -204,15 +199,15 @@ public ExprValue visit(Object obj) { @Override public ExprValue visitObject(JSONObject jsonObject) { - return fromObjectValue(jsonObject.keySet().stream().collect(Collectors.toMap( - key -> key, - key -> visit(jsonObject.get(key)) - ))); + return fromObjectValue( + jsonObject.keySet().stream() + .collect(Collectors.toMap(key -> key, key -> visit(jsonObject.get(key))))); } @Override public ExprValue visitArray(JSONArray jsonArray) { - return new ExprArrayValue(stream(spliteratorUnknownSize(jsonArray.iterator(), Spliterator.ORDERED), false) + return new ExprArrayValue( + stream(spliteratorUnknownSize(jsonArray.iterator(), Spliterator.ORDERED), false) .map(this::visit) .collect(Collectors.toList())); } diff --git a/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java b/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java index f142d83e37..f87e873f4a 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java +++ b/spark/src/main/java/org/opensearch/sql/spark/functions/response/JsonVisitor.java @@ -3,14 +3,12 @@ import org.json.JSONArray; import org.json.JSONObject; -/** - * visitor for the generic Json Object arriving from the flint's result payload - */ +/** visitor for the generic Json Object arriving from the flint's result payload */ public interface JsonVisitor { - T visitObject(JSONObject jsonObject); + T visitObject(JSONObject jsonObject); - T visitArray(JSONArray jsonArray); + T visitArray(JSONArray jsonArray); - T visitPrimitive(Object primitive); + T visitPrimitive(Object primitive); } diff --git a/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java b/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java index 1f11b3c66e..3e783bf67c 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java +++ b/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java @@ -1,5 +1,11 @@ package org.opensearch.sql.spark.functions.response; +import static java.lang.String.format; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.model.ExprTupleValue.fromExprValueMap; + +import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprBooleanValue; @@ -9,18 +15,13 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.executor.ExecutionEngine; -import java.util.List; -import java.util.Map; - -import static java.lang.String.format; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.data.model.ExprTupleValue.fromExprValueMap; - class DefaultSparkSqlFunctionResponseHandleTest { - @Test - void testJsonObject() { - JSONObject jsonObject = new JSONObject("{" + @Test + void testJsonObject() { + JSONObject jsonObject = + new JSONObject( + "{" + "\"data\": {" + " \"applicationId\": \"some_spark_application_id\"," + " \"schema\": [" @@ -34,212 +35,338 @@ void testJsonObject() { + " ]" + "}" + "}"); - DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); - assertEquals(new ExecutionEngine.Schema(List.of( - new ExecutionEngine.Schema.Column("column1", "column1", ExprCoreType.STRING), - new ExecutionEngine.Schema.Column("column2", "column2", ExprCoreType.INTEGER), - new ExecutionEngine.Schema.Column("column3", "column3", ExprCoreType.BOOLEAN))), - handle.schema()); - - assertEquals(true, handle.hasNext()); - assertEquals(fromExprValueMap(Map.of("column1", new ExprStringValue("value1"), "column2", new ExprIntegerValue(123), "column3", ExprBooleanValue.of(true))), handle.next()); - assertEquals(fromExprValueMap(Map.of("column1", new ExprStringValue("value2"), "column2", new ExprIntegerValue(456), "column3", ExprBooleanValue.of(false))), handle.next()); - assertEquals(false, handle.hasNext()); - } - - @Test - void testJsonArrayObject() { - //region attributes - String data = " { 'attributes': [\n" + - " {\n" + - " 'key': 'telemetry.sdk.language',\n" + - " 'value': {\n" + - " 'stringValue': 'python'\n" + - " }\n" + - " },\n" + - " {\n" + - " 'key': 'telemetry.sdk.name',\n" + - " 'value': {\n" + - " 'stringValue': 'opentelemetry'\n" + - " }\n" + - " },\n" + - " {\n" + - " 'key': 'telemetry.sdk.version',\n" + - " 'value': {\n" + - " 'stringValue': '1.19.0'\n" + - " }\n" + - " },\n" + - " {\n" + - " 'key': 'service.namespace',\n" + - " 'value': {\n" + - " 'stringValue': 'opentelemetry-demo'\n" + - " }\n" + - " },\n" + - " {\n" + - " 'key': 'service.name',\n" + - " 'value': {\n" + - " 'stringValue': 'recommendationservice'\n" + - " }\n" + - " },\n" + - " {\n" + - " 'key': 'telemetry.auto.version',\n" + - " 'value': {\n" + - " 'stringValue': '0.40b0'\n" + - " }\n" + - " }\n" + - " ]\n }"; - //endregion attributes - //region schema - String schema = "{'column_name':'attributes','data_type':'array'}"; - //endregion schema - - JSONObject jsonObject = new JSONObject(format("{" + "\"data\": "+ - " {\n" + - " \"applicationId\": \"00fd777k3k3ls20p\",\n" + - " \"schema\": [\n" + - " %s" + - " ],\n" + - " \"result\": [\n" + - " %s" + - " ],\n" + - " }\n" + - " }\n" + - " ]\n" + - " }" + - "}",schema,data)); - - DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); - assertEquals(new ExecutionEngine.Schema(List.of( - new ExecutionEngine.Schema.Column("attributes", "attributes", ExprCoreType.ARRAY))), - handle.schema()); - - assertEquals(true, handle.hasNext()); - ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); - assertEquals(1,tupleValue.tupleValue().size()); - assertEquals(true,tupleValue.tupleValue().get("attributes").isArray()); - assertEquals(6, tupleValue.tupleValue().get("attributes").arrayValue().size()); - assertEquals(false, handle.hasNext()); - } - - @Test - void testJsonNestedObject() { - //region resourceSpans - String data = "{\n" + - " 'resourceSpans': {\n" + - " 'scopeSpans': {\n" + - " 'spans': {\n" + - " 'key': 'rpc.system',\n" + - " 'value': {\n" + - " 'stringValue': 'grpc'\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - //endregion resourceSpans - //region schema - String schema = "{'column_name':'resourceSpans','data_type':'struct'}"; - //endregion schema - - JSONObject jsonObject = new JSONObject(format("{" + "\"data\": "+ - " {\n" + - " \"applicationId\": \"00fd777k3k3ls20p\",\n" + - " \"schema\": [\n" + - " %s" + - " ],\n" + - " \"result\": [\n" + - " %s" + - " ],\n" + - " }\n" + - " }\n" + - " ]\n" + - " }" + - "}",schema,data)); - - DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); - assertEquals(new ExecutionEngine.Schema(List.of( - new ExecutionEngine.Schema.Column("resourceSpans", "resourceSpans", ExprCoreType.STRUCT))), - handle.schema()); - - assertEquals(true, handle.hasNext()); - ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); - assertEquals(1,tupleValue.tupleValue().size()); - assertEquals("resourceSpans",tupleValue.tupleValue().keySet().iterator().next()); - assertEquals(1,tupleValue.keyValue("resourceSpans").tupleValue().size()); - assertEquals("scopeSpans",tupleValue.keyValue("resourceSpans").tupleValue().keySet().iterator().next()); - assertEquals("spans",tupleValue.keyValue("resourceSpans").keyValue("scopeSpans").tupleValue().keySet().iterator().next()); - assertEquals(2,tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").tupleValue().keySet().size()); - - assertEquals(false, handle.hasNext()); - - } - @Test - void testJsonNestedObjectArray() { - //region resourceSpans - String data = "{\n" + - " 'resourceSpans': \n" + - " {\n" + - " 'scopeSpans': \n" + - " {\n" + - " 'spans': \n" + - " [\n" + - " {\n" + - " 'attribute': {\n" + - " 'key': 'rpc.system',\n" + - " 'value': {\n" + - " 'stringValue': 'grpc'\n" + - " }\n" + - " }\n" + - " },\n" + - " {\n" + - " 'attribute': {\n" + - " 'key': 'rpc.system',\n" + - " 'value': {\n" + - " 'stringValue': 'grpc'\n" + - " }\n" + - " }\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}"; - //endregion resourceSpans - //region schema - String schema = "{'column_name':'resourceSpans','data_type':'struct'}"; - //endregion schema - - JSONObject jsonObject = new JSONObject(format("{" + "\"data\": "+ - " {\n" + - " \"applicationId\": \"00fd777k3k3ls20p\",\n" + - " \"schema\": [\n" + - " %s" + - " ],\n" + - " \"result\": [\n" + - " %s" + - " ],\n" + - " }\n" + - " }\n" + - " ]\n" + - " }" + - "}",schema,data)); - - DefaultSparkSqlFunctionResponseHandle handle = new DefaultSparkSqlFunctionResponseHandle(jsonObject); - assertEquals(new ExecutionEngine.Schema(List.of( - new ExecutionEngine.Schema.Column("resourceSpans", "resourceSpans", ExprCoreType.STRUCT))), - handle.schema()); - - assertEquals(true, handle.hasNext()); - ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); - assertEquals(1,tupleValue.tupleValue().size()); - assertEquals("resourceSpans",tupleValue.tupleValue().keySet().iterator().next()); - assertEquals(1,tupleValue.keyValue("resourceSpans").tupleValue().size()); - assertEquals("scopeSpans",tupleValue.keyValue("resourceSpans").tupleValue().keySet().iterator().next()); - assertEquals("spans",tupleValue.keyValue("resourceSpans").keyValue("scopeSpans").tupleValue().keySet().iterator().next()); - assertEquals(true,tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").isArray()); - assertEquals(2,tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").arrayValue().size()); - assertEquals("attribute",tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").arrayValue().get(0).tupleValue().keySet().iterator().next()); - assertEquals("attribute",tupleValue.tupleValue().values().iterator().next().keyValue("scopeSpans").keyValue("spans").arrayValue().get(1).tupleValue().keySet().iterator().next()); - - assertEquals(false, handle.hasNext()); - } + DefaultSparkSqlFunctionResponseHandle handle = + new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals( + new ExecutionEngine.Schema( + List.of( + new ExecutionEngine.Schema.Column("column1", "column1", ExprCoreType.STRING), + new ExecutionEngine.Schema.Column("column2", "column2", ExprCoreType.INTEGER), + new ExecutionEngine.Schema.Column("column3", "column3", ExprCoreType.BOOLEAN))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + assertEquals( + fromExprValueMap( + Map.of( + "column1", + new ExprStringValue("value1"), + "column2", + new ExprIntegerValue(123), + "column3", + ExprBooleanValue.of(true))), + handle.next()); + assertEquals( + fromExprValueMap( + Map.of( + "column1", + new ExprStringValue("value2"), + "column2", + new ExprIntegerValue(456), + "column3", + ExprBooleanValue.of(false))), + handle.next()); + assertEquals(false, handle.hasNext()); + } + + @Test + void testJsonArrayObject() { + // region attributes + String data = + " { 'attributes': [\n" + + " {\n" + + " 'key': 'telemetry.sdk.language',\n" + + " 'value': {\n" + + " 'stringValue': 'python'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'telemetry.sdk.name',\n" + + " 'value': {\n" + + " 'stringValue': 'opentelemetry'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'telemetry.sdk.version',\n" + + " 'value': {\n" + + " 'stringValue': '1.19.0'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'service.namespace',\n" + + " 'value': {\n" + + " 'stringValue': 'opentelemetry-demo'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'service.name',\n" + + " 'value': {\n" + + " 'stringValue': 'recommendationservice'\n" + + " }\n" + + " },\n" + + " {\n" + + " 'key': 'telemetry.auto.version',\n" + + " 'value': {\n" + + " 'stringValue': '0.40b0'\n" + + " }\n" + + " }\n" + + " ]\n }"; + // endregion attributes + // region schema + String schema = "{'column_name':'attributes','data_type':'array'}"; + // endregion schema + + JSONObject jsonObject = + new JSONObject( + format( + "{" + + "\"data\": " + + " {\n" + + " \"applicationId\": \"00fd777k3k3ls20p\",\n" + + " \"schema\": [\n" + + " %s" + + " ],\n" + + " \"result\": [\n" + + " %s" + + " ],\n" + + " }\n" + + " }\n" + + " ]\n" + + " }" + + "}", + schema, data)); + + DefaultSparkSqlFunctionResponseHandle handle = + new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals( + new ExecutionEngine.Schema( + List.of( + new ExecutionEngine.Schema.Column("attributes", "attributes", ExprCoreType.ARRAY))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); + assertEquals(1, tupleValue.tupleValue().size()); + assertEquals(true, tupleValue.tupleValue().get("attributes").isArray()); + assertEquals(6, tupleValue.tupleValue().get("attributes").arrayValue().size()); + assertEquals(false, handle.hasNext()); + } + + @Test + void testJsonNestedObject() { + // region resourceSpans + String data = + "{\n" + + " 'resourceSpans': {\n" + + " 'scopeSpans': {\n" + + " 'spans': {\n" + + " 'key': 'rpc.system',\n" + + " 'value': {\n" + + " 'stringValue': 'grpc'\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + // endregion resourceSpans + // region schema + String schema = "{'column_name':'resourceSpans','data_type':'struct'}"; + // endregion schema + + JSONObject jsonObject = + new JSONObject( + format( + "{" + + "\"data\": " + + " {\n" + + " \"applicationId\": \"00fd777k3k3ls20p\",\n" + + " \"schema\": [\n" + + " %s" + + " ],\n" + + " \"result\": [\n" + + " %s" + + " ],\n" + + " }\n" + + " }\n" + + " ]\n" + + " }" + + "}", + schema, data)); + + DefaultSparkSqlFunctionResponseHandle handle = + new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals( + new ExecutionEngine.Schema( + List.of( + new ExecutionEngine.Schema.Column( + "resourceSpans", "resourceSpans", ExprCoreType.STRUCT))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); + assertEquals(1, tupleValue.tupleValue().size()); + assertEquals("resourceSpans", tupleValue.tupleValue().keySet().iterator().next()); + assertEquals(1, tupleValue.keyValue("resourceSpans").tupleValue().size()); + assertEquals( + "scopeSpans", tupleValue.keyValue("resourceSpans").tupleValue().keySet().iterator().next()); + assertEquals( + "spans", + tupleValue + .keyValue("resourceSpans") + .keyValue("scopeSpans") + .tupleValue() + .keySet() + .iterator() + .next()); + assertEquals( + 2, + tupleValue + .tupleValue() + .values() + .iterator() + .next() + .keyValue("scopeSpans") + .keyValue("spans") + .tupleValue() + .keySet() + .size()); + + assertEquals(false, handle.hasNext()); + } + + @Test + void testJsonNestedObjectArray() { + // region resourceSpans + String data = + "{\n" + + " 'resourceSpans': \n" + + " {\n" + + " 'scopeSpans': \n" + + " {\n" + + " 'spans': \n" + + " [\n" + + " {\n" + + " 'attribute': {\n" + + " 'key': 'rpc.system',\n" + + " 'value': {\n" + + " 'stringValue': 'grpc'\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " 'attribute': {\n" + + " 'key': 'rpc.system',\n" + + " 'value': {\n" + + " 'stringValue': 'grpc'\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + // endregion resourceSpans + // region schema + String schema = "{'column_name':'resourceSpans','data_type':'struct'}"; + // endregion schema + + JSONObject jsonObject = + new JSONObject( + format( + "{" + + "\"data\": " + + " {\n" + + " \"applicationId\": \"00fd777k3k3ls20p\",\n" + + " \"schema\": [\n" + + " %s" + + " ],\n" + + " \"result\": [\n" + + " %s" + + " ],\n" + + " }\n" + + " }\n" + + " ]\n" + + " }" + + "}", + schema, data)); + + DefaultSparkSqlFunctionResponseHandle handle = + new DefaultSparkSqlFunctionResponseHandle(jsonObject); + assertEquals( + new ExecutionEngine.Schema( + List.of( + new ExecutionEngine.Schema.Column( + "resourceSpans", "resourceSpans", ExprCoreType.STRUCT))), + handle.schema()); + + assertEquals(true, handle.hasNext()); + ExprTupleValue tupleValue = (ExprTupleValue) handle.next(); + assertEquals(1, tupleValue.tupleValue().size()); + assertEquals("resourceSpans", tupleValue.tupleValue().keySet().iterator().next()); + assertEquals(1, tupleValue.keyValue("resourceSpans").tupleValue().size()); + assertEquals( + "scopeSpans", tupleValue.keyValue("resourceSpans").tupleValue().keySet().iterator().next()); + assertEquals( + "spans", + tupleValue + .keyValue("resourceSpans") + .keyValue("scopeSpans") + .tupleValue() + .keySet() + .iterator() + .next()); + assertEquals( + true, + tupleValue + .tupleValue() + .values() + .iterator() + .next() + .keyValue("scopeSpans") + .keyValue("spans") + .isArray()); + assertEquals( + 2, + tupleValue + .tupleValue() + .values() + .iterator() + .next() + .keyValue("scopeSpans") + .keyValue("spans") + .arrayValue() + .size()); + assertEquals( + "attribute", + tupleValue + .tupleValue() + .values() + .iterator() + .next() + .keyValue("scopeSpans") + .keyValue("spans") + .arrayValue() + .get(0) + .tupleValue() + .keySet() + .iterator() + .next()); + assertEquals( + "attribute", + tupleValue + .tupleValue() + .values() + .iterator() + .next() + .keyValue("scopeSpans") + .keyValue("spans") + .arrayValue() + .get(1) + .tupleValue() + .keySet() + .iterator() + .next()); + + assertEquals(false, handle.hasNext()); + } } From 381214dc8ec4fd536acffaf1375c5cec2c7dedca Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 26 Oct 2023 22:08:53 -0700 Subject: [PATCH 08/11] fix coverage & additional style Signed-off-by: YANGDB --- .../sql/data/model/ExprArrayValue.java | 5 +- .../sql/data/model/ExprTupleValue.java | 30 ++++---- .../sql/data/model/ExprArrayValueTest.java | 76 +++++++++++++++++++ .../sql/data/model/ExprTupleValueTest.java | 27 ++++++- ...ultSparkSqlFunctionResponseHandleTest.java | 42 +++++----- 5 files changed, 143 insertions(+), 37 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java index 02ae1d16cf..240284173c 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprArrayValue.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -33,7 +34,7 @@ public ExprType type() { @Override public String stringValue() { - return value.stream().map(Object::toString).reduce("", String::concat); + return value.stream().map(ExprValue::stringValue).collect(Collectors.joining(",")); } @Override @@ -43,7 +44,7 @@ public List arrayValue() { @Override public String toString() { - return String.format("\"%s\"", stringValue()); + return String.format("%s", stringValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java index 856075bed8..3ca784d73f 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java @@ -5,7 +5,6 @@ package org.opensearch.sql.data.model; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; @@ -72,20 +71,25 @@ public ExprValue keyValue(String key) { public boolean equal(ExprValue o) { if (!(o instanceof ExprTupleValue)) { return false; - } else { - ExprTupleValue other = (ExprTupleValue) o; - Iterator> thisIterator = this.valueMap.entrySet().iterator(); - Iterator> otherIterator = other.valueMap.entrySet().iterator(); - while (thisIterator.hasNext() && otherIterator.hasNext()) { - Entry thisEntry = thisIterator.next(); - Entry otherEntry = otherIterator.next(); - if (!(thisEntry.getKey().equals(otherEntry.getKey()) - && thisEntry.getValue().equals(otherEntry.getValue()))) { - return false; - } + } + + ExprTupleValue other = (ExprTupleValue) o; + if (this.valueMap.size() != other.valueMap.size()) { + return false; + } + + for (Map.Entry entry : this.valueMap.entrySet()) { + String key = entry.getKey(); + ExprValue value = entry.getValue(); + if (!other.valueMap.containsKey(key)) { + return false; + } + ExprValue otherValue = other.valueMap.get(key); + if (!value.equals(otherValue)) { + return false; } - return !(thisIterator.hasNext() || otherIterator.hasNext()); } + return true; } /** Only compare the size of the map. */ diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java new file mode 100644 index 0000000000..dbc75c4b77 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java @@ -0,0 +1,76 @@ +package org.opensearch.sql.data.model; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.type.ExprCoreType; + +public class ExprArrayValueTest { + @Test + public void testIsArray() { + ExprArrayValue exprArrayValue = new ExprArrayValue(Arrays.asList(new ExprStringValue("test"))); + assertTrue(exprArrayValue.isArray()); + } + + @Test + public void testValue() { + List value = + Arrays.asList(new ExprStringValue("test1"), new ExprStringValue("test2")); + ExprArrayValue exprArrayValue = new ExprArrayValue(value); + assertEquals(value, exprArrayValue.value()); + } + + @Test + public void testType() { + ExprArrayValue exprArrayValue = new ExprArrayValue(Arrays.asList(new ExprStringValue("test"))); + assertEquals(ExprCoreType.ARRAY, exprArrayValue.type()); + } + + @Test + public void testStringValue() { + ExprArrayValue exprArrayValue = + new ExprArrayValue( + Arrays.asList(new ExprStringValue("test1"), new ExprStringValue("test2"))); + assertEquals("test1,test2", exprArrayValue.stringValue()); + } + + @Test + public void testArrayValue() { + List value = + Arrays.asList(new ExprStringValue("test1"), new ExprStringValue("test2")); + ExprArrayValue exprArrayValue = new ExprArrayValue(value); + assertEquals(value, exprArrayValue.arrayValue()); + } + + @Test + public void testToString() { + ExprArrayValue exprArrayValue = new ExprArrayValue(List.of(new ExprStringValue("test"))); + assertEquals("test", exprArrayValue.toString()); + } + + @Test + public void testCompare() { + ExprArrayValue exprArrayValue1 = new ExprArrayValue(Arrays.asList(new ExprStringValue("a"))); + ExprArrayValue exprArrayValue2 = new ExprArrayValue(Arrays.asList(new ExprStringValue("b"))); + assertThat(exprArrayValue1.compare(exprArrayValue2), lessThan(0)); + } + + @Test + public void testEqual() { + ExprArrayValue exprArrayValue1 = new ExprArrayValue(Arrays.asList(new ExprStringValue("test"))); + ExprArrayValue exprArrayValue2 = new ExprArrayValue(Arrays.asList(new ExprStringValue("test"))); + assertTrue(exprArrayValue1.equal(exprArrayValue2)); + } + + @Test + public void testHashCode() { + ExprArrayValue exprArrayValue = new ExprArrayValue(List.of(new ExprStringValue("test"))); + assertEquals(exprArrayValue.hashCode(), Objects.hashCode("test")); + } +} diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprTupleValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprTupleValueTest.java index 567e1e78db..816955aa8b 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/ExprTupleValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprTupleValueTest.java @@ -10,9 +10,11 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprTupleValue.fromExprValueMap; import static org.opensearch.sql.utils.ComparisonUtil.compare; import com.google.common.collect.ImmutableMap; +import java.util.Map; import org.junit.jupiter.api.Test; import org.opensearch.sql.exception.ExpressionEvaluationException; @@ -30,6 +32,27 @@ public void tuple_compare_int() { assertFalse(tupleValue.equals(intValue)); } + @Test + public void compare_tuple_with_same_key_different_order() { + assertEquals( + fromExprValueMap( + Map.of( + "column1", + new ExprStringValue("value1"), + "column2", + new ExprIntegerValue(123), + "column3", + ExprBooleanValue.of(true))), + fromExprValueMap( + Map.of( + "column2", + new ExprIntegerValue(123), + "column1", + new ExprStringValue("value1"), + "column3", + ExprBooleanValue.of(true)))); + } + @Test public void compare_tuple_with_different_key() { ExprValue tupleValue1 = ExprValueUtils.tupleValue(ImmutableMap.of("value", 2)); @@ -44,8 +67,8 @@ public void compare_tuple_with_different_size() { ExprValue tupleValue1 = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2)); ExprValue tupleValue2 = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2, "float_value", 1f)); - assertFalse(tupleValue1.equals(tupleValue2)); - assertFalse(tupleValue2.equals(tupleValue1)); + assertNotEquals(tupleValue1, tupleValue2); + assertNotEquals(tupleValue2, tupleValue1); } @Test diff --git a/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java b/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java index 3e783bf67c..e0a331e876 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java +++ b/spark/src/test/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java @@ -2,6 +2,8 @@ import static java.lang.String.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprTupleValue.fromExprValueMap; import java.util.List; @@ -45,28 +47,28 @@ void testJsonObject() { new ExecutionEngine.Schema.Column("column3", "column3", ExprCoreType.BOOLEAN))), handle.schema()); - assertEquals(true, handle.hasNext()); - assertEquals( + assertTrue(handle.hasNext()); + assertTrue( fromExprValueMap( - Map.of( - "column1", - new ExprStringValue("value1"), - "column2", - new ExprIntegerValue(123), - "column3", - ExprBooleanValue.of(true))), - handle.next()); - assertEquals( + Map.of( + "column1", + new ExprStringValue("value1"), + "column2", + new ExprIntegerValue(123), + "column3", + ExprBooleanValue.of(true))) + .equal(handle.next())); + assertTrue( fromExprValueMap( - Map.of( - "column1", - new ExprStringValue("value2"), - "column2", - new ExprIntegerValue(456), - "column3", - ExprBooleanValue.of(false))), - handle.next()); - assertEquals(false, handle.hasNext()); + Map.of( + "column1", + new ExprStringValue("value2"), + "column2", + new ExprIntegerValue(456), + "column3", + ExprBooleanValue.of(false))) + .equal(handle.next())); + assertFalse(handle.hasNext()); } @Test From 2353a14c956581ed1ecf8adfbb34453189583e07 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 26 Oct 2023 22:46:41 -0700 Subject: [PATCH 09/11] fix coverage Signed-off-by: YANGDB --- .../org/opensearch/sql/data/model/ExprArrayValueTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java index dbc75c4b77..f5d53be790 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java @@ -3,6 +3,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.lessThan; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; @@ -12,10 +13,14 @@ import org.opensearch.sql.data.type.ExprCoreType; public class ExprArrayValueTest { + @Test + public void testIsArrayFalse() { + assertFalse(new ExprStringValue("test").isArray()); + } @Test public void testIsArray() { ExprArrayValue exprArrayValue = new ExprArrayValue(Arrays.asList(new ExprStringValue("test"))); - assertTrue(exprArrayValue.isArray()); + assertFalse(exprArrayValue.isArray()); } @Test From 92bce290b6d838d234c7617ac282c7d0303bd3e1 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 26 Oct 2023 22:48:43 -0700 Subject: [PATCH 10/11] make JsonVisitorImpl private Signed-off-by: YANGDB --- .../response/DefaultSparkSqlFunctionResponseHandle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java index faef49c071..dd574b2d30 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java +++ b/spark/src/main/java/org/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandle.java @@ -186,7 +186,7 @@ public ExecutionEngine.Schema schema() { } /** visitor implementation for traversing the json object */ - public static class JsonVisitorImpl implements JsonVisitor { + private static class JsonVisitorImpl implements JsonVisitor { public ExprValue visit(Object obj) { if (obj instanceof JSONObject) { return visitObject((JSONObject) obj); From 13867e51443e386b65d31cd57ae6aacdda7a2e9c Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 27 Oct 2023 12:09:58 -0700 Subject: [PATCH 11/11] update spotlessApply styling issues Signed-off-by: YANGDB --- .../java/org/opensearch/sql/data/model/ExprArrayValueTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java index f5d53be790..e63c287a6b 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/ExprArrayValueTest.java @@ -17,6 +17,7 @@ public class ExprArrayValueTest { public void testIsArrayFalse() { assertFalse(new ExprStringValue("test").isArray()); } + @Test public void testIsArray() { ExprArrayValue exprArrayValue = new ExprArrayValue(Arrays.asList(new ExprStringValue("test")));