diff --git a/core/src/main/java/org/opensearch/sql/expression/ip/IPFunction.java b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunction.java index 8dde5e6108..48738de54c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ip/IPFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunction.java @@ -7,7 +7,9 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.expression.function.FunctionDSL.*; +import static org.opensearch.sql.expression.function.FunctionDSL.define; +import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import inet.ipaddr.AddressStringException; import inet.ipaddr.IPAddressString; diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 2888116848..2412bd9474 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -6,31 +6,86 @@ package org.opensearch.sql.analysis; import static java.util.Collections.emptyList; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction; -import static org.opensearch.sql.ast.dsl.AstDSL.*; -import static org.opensearch.sql.ast.tree.Sort.*; +import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; +import static org.opensearch.sql.ast.dsl.AstDSL.alias; +import static org.opensearch.sql.ast.dsl.AstDSL.argument; +import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.filter; +import static org.opensearch.sql.ast.dsl.AstDSL.filteredAggregate; +import static org.opensearch.sql.ast.dsl.AstDSL.function; +import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.nestedAllTupleFields; +import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; +import static org.opensearch.sql.ast.dsl.AstDSL.span; +import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; +import static org.opensearch.sql.ast.tree.Sort.NullOrder; +import static org.opensearch.sql.ast.tree.Sort.SortOption; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; +import static org.opensearch.sql.ast.tree.Sort.SortOrder; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; -import static org.opensearch.sql.data.type.ExprCoreType.*; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.expression.DSL.literal; -import static org.opensearch.sql.utils.MLCommonsConstants.*; +import static org.opensearch.sql.utils.MLCommonsConstants.ACTION; +import static org.opensearch.sql.utils.MLCommonsConstants.ALGO; +import static org.opensearch.sql.utils.MLCommonsConstants.ASYNC; +import static org.opensearch.sql.utils.MLCommonsConstants.CLUSTERID; +import static org.opensearch.sql.utils.MLCommonsConstants.KMEANS; +import static org.opensearch.sql.utils.MLCommonsConstants.MODELID; +import static org.opensearch.sql.utils.MLCommonsConstants.PREDICT; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIME_FIELD; +import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; +import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.opensearch.sql.ast.dsl.AstDSL; -import org.opensearch.sql.ast.expression.*; -import org.opensearch.sql.ast.tree.*; +import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.HighlightFunction; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.ParseMethod; +import org.opensearch.sql.ast.expression.ScoreFunction; +import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.ast.tree.AD; +import org.opensearch.sql.ast.tree.CloseCursor; +import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.Kmeans; +import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -40,7 +95,16 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.OpenSearchFunctions; import org.opensearch.sql.expression.window.WindowDefinition; -import org.opensearch.sql.planner.logical.*; +import org.opensearch.sql.planner.logical.LogicalAD; +import org.opensearch.sql.planner.logical.LogicalCloseCursor; +import org.opensearch.sql.planner.logical.LogicalFetchCursor; +import org.opensearch.sql.planner.logical.LogicalFilter; +import org.opensearch.sql.planner.logical.LogicalMLCommons; +import org.opensearch.sql.planner.logical.LogicalPaginate; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalPlanDSL; +import org.opensearch.sql.planner.logical.LogicalProject; +import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.physical.datasource.DataSourceTable; class AnalyzerTest extends AnalyzerTestBase { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java index b40e180816..897f49cfee 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateSubAndSubDateTest.java @@ -10,7 +10,13 @@ import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import java.time.*; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.Period; +import java.time.ZoneOffset; import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; diff --git a/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java index 42959a7f90..ccee1783d1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/ip/IPFunctionTest.java @@ -5,7 +5,9 @@ package org.opensearch.sql.expression.ip; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java index d2542ffe9d..cd64b7359b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/IPFunctionIT.java @@ -6,7 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOG; -import static org.opensearch.sql.util.MatcherUtils.*; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; import org.json.JSONObject; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 0336c6ccac..9908439886 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -5,17 +5,37 @@ package org.opensearch.sql.opensearch.data.value; -import static org.opensearch.sql.data.type.ExprCoreType.*; -import static org.opensearch.sql.utils.DateTimeFormatters.*; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER; +import static org.opensearch.sql.utils.DateTimeFormatters.STRICT_HOUR_MINUTE_SECOND_FORMATTER; +import static org.opensearch.sql.utils.DateTimeFormatters.STRICT_YEAR_MONTH_DAY_FORMATTER; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; -import java.time.*; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; @@ -23,7 +43,21 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; -import org.opensearch.sql.data.model.*; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprFloatValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprShortValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTimeValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; @@ -37,34 +71,8 @@ /** Construct ExprValue from OpenSearch response. */ public class OpenSearchExprValueFactory { - /** The Mapping of Field and ExprType. */ - private final Map typeMapping; - - /** Whether to support nested value types (such as arrays) */ - private final boolean fieldTypeTolerance; - - /** - * Extend existing mapping by new data without overwrite. Called from aggregation only {@see - * AggregationQueryBuilder#buildTypeMapping}. - * - * @param typeMapping A data type mapping produced by aggregation. - */ - public void extendTypeMapping(Map typeMapping) { - for (var field : typeMapping.keySet()) { - // Prevent overwriting, because aggregation engine may be not aware - // of all niceties of all types. - if (!this.typeMapping.containsKey(field)) { - this.typeMapping.put(field, typeMapping.get(field)); - } - } - } - - @Getter @Setter private OpenSearchAggregationResponseParser parser; - private static final String TOP_PATH = ""; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final Map> typeActionMap = new ImmutableMap.Builder>() .put( @@ -108,6 +116,14 @@ public void extendTypeMapping(Map typeMapping) { (c, dt) -> new OpenSearchExprBinaryValue(c.stringValue())) .build(); + /** The Mapping of Field and ExprType. */ + private final Map typeMapping; + + /** Whether to support nested value types (such as arrays) */ + private final boolean fieldTypeTolerance; + + @Getter @Setter private OpenSearchAggregationResponseParser parser; + /** Constructor of OpenSearchExprValueFactory. */ public OpenSearchExprValueFactory( Map typeMapping, boolean fieldTypeTolerance) { @@ -115,78 +131,6 @@ public OpenSearchExprValueFactory( this.fieldTypeTolerance = fieldTypeTolerance; } - /** - * - * - * 
-   * The struct construction has the following assumption:
-   *  1. The field has OpenSearch Object data type.
-   *     See 
-   *       docs
-   *  2. The deeper field is flattened in the typeMapping. e.g.
-   *     { "employ",       "STRUCT"  }
-   *     { "employ.id",    "INTEGER" }
-   *     { "employ.state", "STRING"  }
-   *
- */ - public ExprValue construct(String jsonString, boolean supportArrays) { - try { - return parse( - new OpenSearchJsonContent(OBJECT_MAPPER.readTree(jsonString)), - TOP_PATH, - Optional.of(STRUCT), - fieldTypeTolerance || supportArrays); - } catch (JsonProcessingException e) { - throw new IllegalStateException(String.format("invalid json: %s.", jsonString), e); - } - } - - /** - * Construct ExprValue from field and its value object. Throw exception if trying to construct - * from field of unsupported type.
- * Todo, add IP, GeoPoint support after we have function implementation around it. - * - * @param field field name - * @param value value object - * @return ExprValue - */ - public ExprValue construct(String field, Object value, boolean supportArrays) { - return parse(new ObjectContent(value), field, type(field), supportArrays); - } - - private ExprValue parse( - Content content, String field, Optional fieldType, boolean supportArrays) { - if (content.isNull() || !fieldType.isPresent()) { - return ExprNullValue.of(); - } - - final ExprType type = fieldType.get(); - - if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))) { - return parseGeoPoint(content, supportArrays); - } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) - || content.isArray()) { - return parseArray(content, field, type, supportArrays); - } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) - || type == STRUCT) { - return parseStruct(content, field, supportArrays); - } else if (typeActionMap.containsKey(type)) { - return typeActionMap.get(type).apply(content, type); - } else { - throw new IllegalStateException( - String.format( - "Unsupported type: %s for value: %s.", type.typeName(), content.objectValue())); - } - } - - /** - * In OpenSearch, it is possible field doesn't have type definition in mapping. but has empty - * value. For example, {"empty_field": []}. - */ - private Optional type(String field) { - return Optional.ofNullable(typeMapping.get(field)); - } - /** * Parse value with the first matching formatter into {@link ExprValue} with corresponding {@link * ExprCoreType}. @@ -274,6 +218,94 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) return new ExprTimestampValue((Instant) value.objectValue()); } + /** + * Extend existing mapping by new data without overwrite. Called from aggregation only {@see + * AggregationQueryBuilder#buildTypeMapping}. + * + * @param typeMapping A data type mapping produced by aggregation. + */ + public void extendTypeMapping(Map typeMapping) { + for (var field : typeMapping.keySet()) { + // Prevent overwriting, because aggregation engine may be not aware + // of all niceties of all types. + if (!this.typeMapping.containsKey(field)) { + this.typeMapping.put(field, typeMapping.get(field)); + } + } + } + + /** + * + * + * 
+   * The struct construction has the following assumption:
+   *  1. The field has OpenSearch Object data type.
+   *     See 
+   *       docs
+   *  2. The deeper field is flattened in the typeMapping. e.g.
+   *     { "employ",       "STRUCT"  }
+   *     { "employ.id",    "INTEGER" }
+   *     { "employ.state", "STRING"  }
+   *
+ */ + public ExprValue construct(String jsonString, boolean supportArrays) { + try { + return parse( + new OpenSearchJsonContent(OBJECT_MAPPER.readTree(jsonString)), + TOP_PATH, + Optional.of(STRUCT), + fieldTypeTolerance || supportArrays); + } catch (JsonProcessingException e) { + throw new IllegalStateException(String.format("invalid json: %s.", jsonString), e); + } + } + + /** + * Construct ExprValue from field and its value object. Throw exception if trying to construct + * from field of unsupported type.
+ * Todo, add IP, GeoPoint support after we have function implementation around it. + * + * @param field field name + * @param value value object + * @return ExprValue + */ + public ExprValue construct(String field, Object value, boolean supportArrays) { + return parse(new ObjectContent(value), field, type(field), supportArrays); + } + + private ExprValue parse( + Content content, String field, Optional fieldType, boolean supportArrays) { + if (content.isNull() || !fieldType.isPresent()) { + return ExprNullValue.of(); + } + + final ExprType type = fieldType.get(); + + if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))) { + return parseGeoPoint(content, supportArrays); + } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) + || content.isArray()) { + return parseArray(content, field, type, supportArrays); + } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) + || type == STRUCT) { + return parseStruct(content, field, supportArrays); + } else if (typeActionMap.containsKey(type)) { + return typeActionMap.get(type).apply(content, type); + } else { + throw new IllegalStateException( + String.format( + "Unsupported type: %s for value: %s.", type.typeName(), content.objectValue())); + } + } + + /** + * In OpenSearch, it is possible field doesn't have type definition in mapping. but has empty + * value. For example, {"empty_field": []}. + */ + private Optional type(String field) { + return Optional.ofNullable(typeMapping.get(field)); + } + /** * Parse struct content. *