From f2df7afd76276ad43010e323c8830ac47012cf74 Mon Sep 17 00:00:00 2001 From: Manasvini B S Date: Tue, 18 Jun 2024 12:25:40 -0700 Subject: [PATCH] Add support for custom date format and openSearch date format for date fields as part of Lucene query Github Issue - https://github.com/opensearch-project/sql/issues/2700 Signed-off-by: Manasvini B S --- core/build.gradle | 1 + .../sql/data/model/ExprDateValue.java | 45 +- .../sql/data/model/ExprTimeValue.java | 43 +- .../sql/data/model/ExprTimestampValue.java | 66 ++- .../sql/utils/DateTimeFormatters.java | 18 + .../sql/data/model/DateTimeValueTest.java | 385 +++++++++++++++++- .../datetime/DateTimeFunctionTest.java | 1 - .../runner/connection/JDBCConnection.java | 46 ++- .../sql/legacy/AggregationExpressionIT.java | 6 +- .../org/opensearch/sql/ppl/DataTypeIT.java | 4 +- .../opensearch/sql/ppl/FieldsCommandIT.java | 2 +- .../opensearch/sql/ppl/StatsCommandIT.java | 4 +- .../opensearch/sql/ppl/SystemFunctionIT.java | 11 +- .../opensearch/sql/sql/DateTimeFormatsIT.java | 35 +- .../sql/sql/DateTimeFunctionIT.java | 23 +- .../org/opensearch/sql/sql/JdbcFormatIT.java | 2 +- .../opensearch/sql/sql/SystemFunctionIT.java | 11 +- .../data/type/OpenSearchDataType.java | 16 +- .../data/type/OpenSearchDateType.java | 11 + .../value/OpenSearchExprValueFactory.java | 7 +- .../dsl/BucketAggregationBuilder.java | 7 +- .../script/filter/lucene/LuceneQuery.java | 29 +- .../script/filter/lucene/RangeQuery.java | 6 +- .../script/filter/lucene/TermQuery.java | 5 +- .../data/type/OpenSearchDataTypeTest.java | 14 +- .../data/type/OpenSearchDateTypeTest.java | 57 ++- .../storage/OpenSearchIndexTest.java | 2 +- .../script/filter/FilterQueryBuilderTest.java | 11 +- 28 files changed, 731 insertions(+), 137 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 655e7d92c2..0c44b0780d 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -56,6 +56,7 @@ dependencies { api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' + api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" api project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.9.3') diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index c36cd3ea6d..92883e604a 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -15,10 +15,15 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.util.List; import lombok.RequiredArgsConstructor; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.utils.DateTimeFormatters; /** Expression Date Value. */ @RequiredArgsConstructor @@ -26,10 +31,41 @@ public class ExprDateValue extends AbstractExprValue { private final LocalDate date; + /** Formatted date using user defined/default format */ + private String formattedDate; + /** Constructor of ExprDateValue. */ public ExprDateValue(String date) { + this(date, List.of()); + } + + /** Constructor of ExprDateValue to support custom/OpenSearch date formats in mappings. */ + public ExprDateValue(String date, List dateFormatters) { try { - this.date = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); + LocalDate localDate = null; + // check if dateFormatters are empty, then set default ones + if (dateFormatters == null || dateFormatters.isEmpty()) { + dateFormatters = DateTimeFormatters.initializeDateFormatters(); + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(date); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + localDate = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalDate(); + + this.formattedDate = formatter.format(accessor); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + if (localDate == null) { + // Default constructor behavior, parse using java DateTimeFormatter + localDate = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); + } + this.date = localDate; + } catch (DateTimeParseException e) { throw new SemanticCheckException( String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", date)); @@ -38,7 +74,10 @@ public ExprDateValue(String date) { @Override public String value() { - return DateTimeFormatter.ISO_LOCAL_DATE.format(date); + if (this.formattedDate == null) { + return DateTimeFormatter.ISO_LOCAL_DATE.format(date); + } + return this.formattedDate; } @Override @@ -68,7 +107,7 @@ public boolean isDateTime() { @Override public String toString() { - return String.format("DATE '%s'", value()); + return String.format("DATE '%s'", DateTimeFormatter.ISO_LOCAL_DATE.format(date)); } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index 6b5a4a7c48..ec32e64a7d 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -14,12 +14,17 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionProperties; +import org.opensearch.sql.utils.DateTimeFormatters; /** Expression Time Value. */ @RequiredArgsConstructor @@ -27,10 +32,39 @@ public class ExprTimeValue extends AbstractExprValue { private final LocalTime time; + /** Formatted time using user defined/default format */ + private String formattedTime; + /** Constructor of ExprTimeValue. */ public ExprTimeValue(String time) { + this(time, List.of()); + } + + /** Constructor of ExprTimeValue to support custom/OpenSearch date formats in mappings. */ + public ExprTimeValue(String time, List dateFormatters) { try { - this.time = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); + LocalTime localTime = null; + // check if dateFormatters are empty, then set default ones + if (dateFormatters == null || dateFormatters.isEmpty()) { + dateFormatters = DateTimeFormatters.initializeDateFormatters(); + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(time); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + localTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalTime(); + this.formattedTime = formatter.format(accessor); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + if (localTime == null) { + // Default constructor behavior, parse using java DateTimeFormatter + localTime = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); + } + this.time = localTime; } catch (DateTimeParseException e) { throw new SemanticCheckException( String.format("time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", time)); @@ -39,7 +73,10 @@ public ExprTimeValue(String time) { @Override public String value() { - return ISO_LOCAL_TIME.format(time); + if (this.formattedTime == null) { + return ISO_LOCAL_TIME.format(time); + } + return this.formattedTime; } @Override @@ -67,7 +104,7 @@ public boolean isDateTime() { @Override public String toString() { - return String.format("TIME '%s'", value()); + return String.format("TIME '%s'", ISO_LOCAL_TIME.format(time)); } @Override diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java index e103dc7253..3176a96f5f 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java @@ -5,21 +5,26 @@ package org.opensearch.sql.data.model; -import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS; -import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_WITHOUT_NANO; +import static org.opensearch.sql.utils.DateTimeFormatters.*; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAccessor; +import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.utils.DateTimeFormatters; /** Expression Timestamp Value. */ @RequiredArgsConstructor @@ -27,13 +32,44 @@ public class ExprTimestampValue extends AbstractExprValue { private final Instant timestamp; + /** Formatted dateTime using user defined/default format */ + private String formattedDateTime; + /** Constructor. */ public ExprTimestampValue(String timestamp) { + this(timestamp, List.of()); + } + + /** + * Constructor of ExprTimestampValue to support custom/OpenSearch dateTime formats in mappings. + */ + public ExprTimestampValue(String timestamp, List dateFormatters) { try { - this.timestamp = - LocalDateTime.parse(timestamp, DATE_TIME_FORMATTER_VARIABLE_NANOS) - .atZone(ZoneOffset.UTC) - .toInstant(); + Instant localDateTime = null; + // check if dateFormatters are empty, then set default ones + if (dateFormatters == null || dateFormatters.isEmpty()) { + dateFormatters = DateTimeFormatters.initializeDateFormatters(); + } + // parse using OpenSearch DateFormatters + for (DateFormatter formatter : dateFormatters) { + try { + TemporalAccessor accessor = formatter.parse(timestamp); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + localDateTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toInstant(); + this.formattedDateTime = formatter.format(accessor); + break; + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + if (localDateTime == null) { + // Default constructor behavior, parse using java DateTimeFormatter + localDateTime = + LocalDateTime.parse(timestamp, DATE_TIME_FORMATTER_VARIABLE_NANOS) + .atZone(ZoneOffset.UTC) + .toInstant(); + } + this.timestamp = localDateTime; } catch (DateTimeParseException e) { throw new SemanticCheckException( String.format( @@ -45,15 +81,23 @@ public ExprTimestampValue(String timestamp) { /** localDateTime Constructor. */ public ExprTimestampValue(LocalDateTime localDateTime) { this.timestamp = localDateTime.atZone(ZoneOffset.UTC).toInstant(); + this.formattedDateTime = null; + } + + public boolean hasNoFormatter() { + return this.formattedDateTime == null; } @Override public String value() { - return timestamp.getNano() == 0 - ? DATE_TIME_FORMATTER_WITHOUT_NANO - .withZone(ZoneOffset.UTC) - .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) - : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); + if (this.formattedDateTime == null) { + return timestamp.getNano() == 0 + ? DATE_TIME_FORMATTER_WITHOUT_NANO + .withZone(ZoneOffset.UTC) + .format(timestamp.truncatedTo(ChronoUnit.SECONDS)) + : DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp); + } + return this.formattedDateTime; } @Override diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 18e6541514..24a3e20349 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -18,8 +18,12 @@ import java.time.format.ResolverStyle; import java.time.format.SignStyle; import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import lombok.experimental.UtilityClass; +import org.opensearch.common.time.DateFormatter; /** DateTimeFormatter. Reference org.opensearch.common.time.DateFormatters. */ @UtilityClass @@ -43,6 +47,9 @@ public class DateTimeFormatters { private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 9; + public static final List OPENSEARCH_DEFAULT_FORMATS = + Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis"); + public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() .appendOffset("+HHmm", "Z") @@ -109,6 +116,9 @@ public class DateTimeFormatters { public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + public static final DateTimeFormatter SQL_LITERAL_DEFAULT_DATE_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ"); + public static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT) @@ -206,4 +216,12 @@ public class DateTimeFormatters { .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) .toFormatter(); + + public static List initializeDateFormatters() { + List dateFormatters = new ArrayList<>(); + for (String pattern : OPENSEARCH_DEFAULT_FORMATS) { + dateFormatters.add(DateFormatter.forPattern(pattern)); + } + return dateFormatters; + } } diff --git a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java index b5a3d61211..88336a58c1 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java @@ -7,16 +7,23 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.Test; +import org.opensearch.common.time.DateFormatter; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionProperties; @@ -103,6 +110,360 @@ public void dateInUnsupportedFormat() { "date:2020-07-07Z in unsupported format, please use 'yyyy-MM-dd'", exception.getMessage()); } + @Test + void testValidTimestampWithCustomFormatter() { + String timestamp = "2021-11-08T17:00:00Z"; + DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); + ExprTimestampValue timestampValue = + new ExprTimestampValue(timestamp, Collections.singletonList(formatter)); + + assertEquals("2021-11-08T17:00:00Z", timestampValue.value()); + assertEquals(LocalDate.parse("2021-11-08"), timestampValue.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), timestampValue.timeValue()); + assertEquals(TIMESTAMP, timestampValue.type()); + assertEquals( + ZonedDateTime.of(LocalDateTime.parse("2021-11-08T17:00:00"), ZoneOffset.UTC).toInstant(), + timestampValue.timestampValue()); + assertEquals("TIMESTAMP '2021-11-08T17:00:00Z'", timestampValue.toString()); + assertEquals( + LocalDateTime.parse("2021-11-08T17:00:00"), + LocalDateTime.ofInstant(timestampValue.timestampValue(), ZoneOffset.UTC)); + assertThrows( + ExpressionEvaluationException.class, + () -> integerValue(1).timestampValue(), + "invalid to get timestampValue from value of type INTEGER"); + } + + @Test + void testValidTimestampWithMultipleFormatters() { + String timestamp = "2021-11-08T17:00:00Z"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd'T'HH:mm:ssX"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); + + ExprTimestampValue timestampValue = + new ExprTimestampValue(timestamp, Arrays.asList(formatter1, formatter2)); + + assertEquals("2021-11-08T17:00:00Z", timestampValue.value()); + assertEquals(LocalDate.parse("2021-11-08"), timestampValue.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), timestampValue.timeValue()); + assertEquals(TIMESTAMP, timestampValue.type()); + + String timestamp2 = "2021/11/08T17:00:00Z"; + + ExprTimestampValue timestampValue2 = + new ExprTimestampValue(timestamp2, Arrays.asList(formatter1, formatter2)); + + assertEquals("2021/11/08T17:00:00Z", timestampValue2.value()); + assertEquals(LocalDate.parse("2021-11-08"), timestampValue2.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), timestampValue2.timeValue()); + assertEquals(TIMESTAMP, timestampValue2.type()); + } + + @Test + void testEmptyFormatterListForTimeStamp() { + String timestamp = "2021-11-08T17:00:00"; + Instant expectedDateTime = + ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + // Test with null formatter list + ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); + assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); + assertEquals("2021-11-08T17:00:00.000Z", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprTimestampValue valueWithEmptyFormatter = + new ExprTimestampValue(timestamp, Collections.emptyList()); + assertEquals(expectedDateTime, valueWithEmptyFormatter.timestampValue()); + assertEquals("2021-11-08T17:00:00.000Z", valueWithEmptyFormatter.value()); + } + + @Test + void testOpenSearchDateTimeNamedFormatter() { + String timestamp = "2019-03-23T21:34:46"; + DateFormatter formatter = DateFormatter.forPattern("strict_date_hour_minute_second"); + ExprTimestampValue value = + new ExprTimestampValue(timestamp, Collections.singletonList(formatter)); + + assertEquals("2019-03-23T21:34:46", value.value()); + assertEquals(LocalDate.parse("2019-03-23"), value.dateValue()); + assertEquals(LocalTime.parse("21:34:46"), value.timeValue()); + assertEquals(TIMESTAMP, value.type()); + } + + @Test + void testInvalidTimestamp() { + String timestamp = "invalid-timestamp"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd'T'HH:mm:ssX"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ssX"); + + List formatters = Arrays.asList(formatter1, formatter2); + + try { + new ExprTimestampValue(timestamp, formatters); + } catch (SemanticCheckException e) { + assertEquals( + String.format( + "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", + timestamp), + e.getMessage()); + } + } + + @Test + void testEpochDateTimeFormatter() { + long epochTimestamp = 1636390800000L; // Corresponds to "2021-11-08T17:00:00Z" + DateFormatter formatter = DateFormatter.forPattern("epoch_millis"); + + ExprTimestampValue value = + new ExprTimestampValue(Long.toString(epochTimestamp), Collections.singletonList(formatter)); + assertEquals("1636390800000", value.value()); + assertEquals(LocalDate.parse("2021-11-08"), value.dateValue()); + assertEquals(LocalTime.parse("17:00:00"), value.timeValue()); + assertEquals(TIMESTAMP, value.type()); + } + + @Test + void testValidTimeStampWithDefaultFormatters() { + String timestamp = "2021-11-08 17:00:00"; + Instant expectedDateTime = + ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + // Test with null formatter list + ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); + assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); + assertEquals("2021-11-08 17:00:00", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprTimestampValue valueWithEmptyFormatter = + new ExprTimestampValue(timestamp, Collections.emptyList()); + assertEquals(expectedDateTime, valueWithEmptyFormatter.timestampValue()); + assertEquals("2021-11-08 17:00:00", valueWithEmptyFormatter.value()); + } + + @Test + void testValidTimeStampHasNoFormatters() { + String timestamp = "2021-11-08 17:00:00"; + Instant expectedDateTime = + ZonedDateTime.of(2021, 11, 8, 17, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + // Test with null formatter list + ExprTimestampValue valueWithNullFormatter = new ExprTimestampValue(timestamp, null); + assertEquals(expectedDateTime, valueWithNullFormatter.timestampValue()); + assertEquals("2021-11-08 17:00:00", valueWithNullFormatter.value()); + assertTrue(valueWithNullFormatter.hasNoFormatter()); + } + + @Test + void testValidDateWithCustomFormatter() { + String dateString = "2021-11-08"; + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); + DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd"); + ExprDateValue value = new ExprDateValue(dateString, Collections.singletonList(formatter)); + assertEquals(expectedDate, value.dateValue()); + assertEquals("2021-11-08", value.value()); + } + + @Test + void testValidDateWithMultipleFormatters() { + String dateString = "2021-11-08"; + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd")); + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); + + ExprDateValue value = new ExprDateValue(dateString, List.of(formatter1, formatter2)); + + assertEquals(expectedDate, value.dateValue()); + assertEquals("2021-11-08", value.value()); + } + + @Test + void testInvalidDate() { + String dateString = "invalid-date"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); + + try { + new ExprDateValue(dateString, List.of(formatter1, formatter2)); + } catch (SemanticCheckException e) { + assertEquals( + String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), + e.getMessage()); + } + } + + @Test + void testEmptyFormatterListForDate() { + String dateString = "2021-11-08"; + LocalDate expectedDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_LOCAL_DATE); + + // Test with null formatter list + ExprDateValue valueWithNullFormatter = new ExprDateValue(dateString, null); + assertEquals(expectedDate, valueWithNullFormatter.dateValue()); + // Formatted to default strict_date_hour_minute_second format + assertEquals("2021-11-08T00:00:00.000Z", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprDateValue valueWithEmptyFormatter = new ExprDateValue(dateString, Collections.emptyList()); + assertEquals(expectedDate, valueWithEmptyFormatter.dateValue()); + // Formatted to default strict_date_hour_minute_second format + assertEquals("2021-11-08T00:00:00.000Z", valueWithEmptyFormatter.value()); + } + + @Test + void testEpochMillisFormat() { + String dateString = "1636358400000"; // Epoch millis for 2021-11-08 + LocalDate expectedDate = LocalDate.of(2021, 11, 8); + + ExprDateValue value = new ExprDateValue(dateString, null); + assertEquals(expectedDate, value.dateValue()); + assertEquals("1636358400000", value.value()); + } + + @Test + void testInvalidDateWithFallbackDefaultFormatter() { + String dateString = "invalid-date"; + DateFormatter formatter1 = DateFormatter.forPattern("yyyy/MM/dd"); + DateFormatter formatter2 = DateFormatter.forPattern("yyyy-MM-dd"); + + // Test with invalid date and fallback formatter also fails + Exception exception = + assertThrows( + SemanticCheckException.class, + () -> { + new ExprDateValue(dateString, List.of(formatter1, formatter2)); + }); + assertEquals( + String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), + exception.getMessage()); + } + + @Test + void testInvalidDateThrowsException() { + String dateString = "invalid-date"; + + Exception exception = + assertThrows( + SemanticCheckException.class, + () -> { + new ExprDateValue(dateString, null); + }); + assertEquals( + String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", dateString), + exception.getMessage()); + } + + @Test + void testValidDateWithDefaultFormatters() { + String dateString = "2021-11-08 00:00:00"; + LocalDate expectedDate = LocalDate.of(2021, 11, 8); + + // Test with null formatter list + ExprDateValue valueWithNullFormatter = new ExprDateValue(dateString, null); + assertEquals(expectedDate, valueWithNullFormatter.dateValue()); + // Formatted to default strict_date_optional_time format + assertEquals("2021-11-08", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprDateValue valueWithEmptyFormatter = new ExprDateValue(dateString, Collections.emptyList()); + assertEquals(expectedDate, valueWithEmptyFormatter.dateValue()); + // Formatted to default strict_date_optional_time format + assertEquals("2021-11-08", valueWithEmptyFormatter.value()); + } + + @Test + void testValidTimeWithCustomFormatter() { + String timeString = "12:10:30.000"; + LocalTime expectedTime = + LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss.SSS")); + DateFormatter formatter = DateFormatter.forPattern("HH:mm:ss.SSS"); + + ExprTimeValue value = new ExprTimeValue(timeString, Collections.singletonList(formatter)); + + assertEquals(expectedTime, value.timeValue()); + assertEquals("12:10:30.000", value.value()); + assertEquals("TIME '12:10:30'", value.toString()); + } + + @Test + void testValidTimeWithMultipleFormatters() { + String timeString = "12:10:30"; + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ofPattern("HH:mm:ss")); + DateFormatter formatter1 = DateFormatter.forPattern("HH:mm:ss.SSS"); + DateFormatter formatter2 = DateFormatter.forPattern("HH:mm:ss"); + + ExprTimeValue value = new ExprTimeValue(timeString, List.of(formatter1, formatter2)); + + assertEquals(expectedTime, value.timeValue()); + assertEquals("12:10:30", value.value()); + } + + @Test + void testInvalidTime() { + String timeString = "invalid-time"; + DateFormatter formatter1 = DateFormatter.forPattern("HH:mm:ss.SSS"); + DateFormatter formatter2 = DateFormatter.forPattern("HH:mm:ss"); + + try { + new ExprTimeValue(timeString, List.of(formatter1, formatter2)); + } catch (SemanticCheckException e) { + assertEquals( + String.format( + "time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", timeString), + e.getMessage()); + } + } + + @Test + void testEmptyFormatterListForTime() { + String timeString = "12:10:30"; + LocalTime expectedTime = LocalTime.parse(timeString, DateTimeFormatter.ISO_LOCAL_TIME); + + // Test with null formatter list + ExprTimeValue valueWithNullFormatter = new ExprTimeValue(timeString, null); + assertEquals(expectedTime, valueWithNullFormatter.timeValue()); + assertEquals("12:10:30", valueWithNullFormatter.value()); + + // Test with empty formatter list + ExprTimeValue valueWithEmptyFormatter = new ExprTimeValue(timeString, Collections.emptyList()); + assertEquals(expectedTime, valueWithEmptyFormatter.timeValue()); + assertEquals("12:10:30", valueWithEmptyFormatter.value()); + } + + @Test + void testEpochTimeFormatter() { + long epochMilli = 1420070400000L; // epoch time in milliseconds + LocalTime expectedTime = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC).toLocalTime(); + DateFormatter epochFormatter = DateFormatter.forPattern("epoch_millis"); + + ExprTimeValue value = + new ExprTimeValue(String.valueOf(epochMilli), Collections.singletonList(epochFormatter)); + + assertEquals(expectedTime, value.timeValue()); + assertEquals(String.valueOf(epochMilli), value.value()); + assertEquals(TIME, value.type()); + assertTrue(value.isDateTime()); + assertEquals("TIME '00:00:00'", value.toString()); + + var exception = assertThrows(ExpressionEvaluationException.class, value::dateValue); + assertEquals("invalid to get dateValue from value of type TIME", exception.getMessage()); + exception = assertThrows(ExpressionEvaluationException.class, value::timestampValue); + assertEquals("invalid to get timestampValue from value of type TIME", exception.getMessage()); + exception = + assertThrows(ExpressionEvaluationException.class, () -> integerValue(1).timeValue()); + assertEquals("invalid to get timeValue from value of type INTEGER", exception.getMessage()); + + var functionProperties = new FunctionProperties(); + var today = LocalDate.now(functionProperties.getQueryStartClock()); + assertEquals(today, value.dateValue(functionProperties)); + assertEquals( + today.atTime(0, 0, 0), + LocalDateTime.ofInstant(value.timestampValue(functionProperties), ZoneOffset.UTC)); + assertEquals( + ZonedDateTime.of(LocalTime.parse("00:00:00").atDate(today), ZoneOffset.UTC).toInstant(), + value.timestampValue(functionProperties)); + } + @Test public void timeInUnsupportedFormat() { SemanticCheckException exception = @@ -113,14 +474,14 @@ public void timeInUnsupportedFormat() { } @Test - public void timestampInUnsupportedFormat() { - SemanticCheckException exception = - assertThrows( - SemanticCheckException.class, () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); - assertEquals( - "timestamp:2020-07-07T01:01:01Z in unsupported format, " - + "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", - exception.getMessage()); + public void timestampSupportedFormat() { + String timestamp = "2020-07-07T01:01:01Z"; + ExprTimestampValue timestampValue = new ExprTimestampValue(timestamp); + + assertEquals("2020-07-07T01:01:01Z", timestampValue.value()); + assertEquals(LocalDate.parse("2020-07-07"), timestampValue.dateValue()); + assertEquals(LocalTime.parse("01:01:01"), timestampValue.timeValue()); + assertEquals(TIMESTAMP, timestampValue.type()); } @Test @@ -134,13 +495,9 @@ public void stringTimestampValue() { assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue()); assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString()); - SemanticCheckException exception = - assertThrows( - SemanticCheckException.class, - () -> new ExprStringValue("2020-07-07T01:01:01Z").timestampValue()); + Instant timestampValue = new ExprStringValue("2020-07-07T01:01:01Z").timestampValue(); assertEquals( - "date:2020-07-07T01:01:01Z in unsupported format, " + "please use 'yyyy-MM-dd'", - exception.getMessage()); + ZonedDateTime.of(2020, 07, 07, 1, 1, 1, 0, ZoneOffset.UTC).toInstant(), timestampValue); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index c820c97196..1ed2a64e76 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1443,7 +1443,6 @@ public void testTimeFormat( private static Stream getInvalidTestDataForTimeFormat() { return Stream.of( Arguments.of(DSL.literal("asdfasdf"), DSL.literal("%f")), - Arguments.of(DSL.literal("12345"), DSL.literal("%h")), Arguments.of(DSL.literal("10:11:61"), DSL.literal("%h")), Arguments.of(DSL.literal("10:61:12"), DSL.literal("%h")), Arguments.of(DSL.literal("61:11:12"), DSL.literal("%h"))); diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java b/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java index 7a67022117..0e60201112 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java @@ -8,15 +8,18 @@ import static java.util.stream.Collectors.joining; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.*; import org.json.JSONObject; import org.opensearch.sql.correctness.runner.resultset.DBResult; import org.opensearch.sql.correctness.runner.resultset.Row; @@ -29,6 +32,20 @@ public class JDBCConnection implements DBConnection { private static final String DOUBLE_QUOTE = "''"; private static final String BACKTICK = "`"; + /** Possible types for date field* */ + private static final Set DATE_TIME_TYPES = ImmutableSet.of("DATE", "TIMESTAMP"); + + /** Formatter used to convert date time from h2 and sqlite to date * */ + private static final DateTimeFormatter DATE_OPTIONAL_TIME_NANO_FORMATTER = + new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd") + .optionalStart() + .appendPattern(" HH:mm:ss") + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .optionalEnd() + .toFormatter(); + /** Database name for display */ private final String databaseName; @@ -170,16 +187,35 @@ private void populateMetaData(ResultSet resultSet, DBResult result) throws SQLEx if (Strings.isNullOrEmpty(colName)) { colName = metaData.getColumnName(i); } - result.addColumn(colName, metaData.getColumnTypeName(i)); + + String type = metaData.getColumnTypeName(i); + // OpenSearch database returns datetime as DATE + if (type.equals("DATE")) { + result.addColumn(colName, mapToJDBCType(type)); + } else { + result.addColumn(colName, type); + } } } private void populateData(ResultSet resultSet, DBResult result) throws SQLException { + while (resultSet.next()) { Row row = new Row(); + ResultSetMetaData metaData = resultSet.getMetaData(); + for (int i = 1; i <= result.columnSize(); i++) { Object value = resultSet.getObject(i); - row.add(resultSet.wasNull() ? null : value); + + // Even though the OpenSearch database returns the full date and time, + // the retrieved object from the result set contains only the date. + // We convert date-time values from other databases to a date format for precise comparison. + if (value != null && DATE_TIME_TYPES.contains(metaData.getColumnTypeName(i))) { + LocalDate dateTime = LocalDate.parse(value.toString(), DATE_OPTIONAL_TIME_NANO_FORMATTER); + row.add(dateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))); + } else { + row.add(resultSet.wasNull() ? null : value); + } } result.addRow(row); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java index 37398220ff..20f04f825e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java @@ -204,7 +204,7 @@ public void groupByDateShouldPass() { Index.BANK.getName())); verifySchema( - response, schema("birthdate", null, "timestamp"), schema("count(*)", "count", "integer")); + response, schema("birthdate", null, "date"), schema("count(*)", "count", "integer")); verifyDataRows(response, rows("2018-06-23 00:00:00", 1)); } @@ -220,9 +220,7 @@ public void groupByDateWithAliasShouldPass() { Index.BANK.getName())); verifySchema( - response, - schema("birthdate", "birth", "timestamp"), - schema("count(*)", "count", "integer")); + response, schema("birthdate", "birth", "date"), schema("count(*)", "count", "integer")); verifyDataRows(response, rows("2018-06-23 00:00:00", 1)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index fe5c2ff270..967e51c722 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -48,8 +48,8 @@ public void test_nonnumeric_data_types() throws IOException { schema("keyword_value", "string"), schema("text_value", "string"), schema("binary_value", "binary"), - schema("date_value", "timestamp"), - schema("date_nanos_value", "timestamp"), + schema("date_value", "date"), + schema("date_nanos_value", "date"), schema("ip_value", "ip"), schema("object_value", "struct"), schema("nested_value", "array"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index e8a287c80e..bb0b2d097a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -56,7 +56,7 @@ public void testFieldsWildCard() throws IOException { public void testSelectDateTypeField() throws IOException { JSONObject result = executeQuery(String.format("source=%s | fields birthdate", TEST_INDEX_BANK)); - verifySchema(result, schema("birthdate", null, "timestamp")); + verifySchema(result, schema("birthdate", null, "date")); verifyDataRows( result, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 2d1cd709e1..f184d0855d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -175,9 +175,7 @@ public void testStatsTimeSpan() throws IOException { executeQuery( String.format("source=%s | stats count() by span(birthdate,1y)", TEST_INDEX_BANK)); verifySchema( - response, - schema("count()", null, "integer"), - schema("span(birthdate,1y)", null, "timestamp")); + response, schema("count()", null, "integer"), schema("span(birthdate,1y)", null, "date")); verifyDataRows(response, rows(2, "2017-01-01 00:00:00"), rows(5, "2018-01-01 00:00:00")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index c1356ce838..1aa5e6b3b5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -82,15 +82,6 @@ public void typeof_opensearch_types() throws IOException { TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, - rows( - "TEXT", - "TIMESTAMP", - "TIMESTAMP", - "BOOLEAN", - "OBJECT", - "KEYWORD", - "IP", - "BINARY", - "GEO_POINT")); + rows("TEXT", "DATE", "DATE", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 13c2eecd56..48798d4ac5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -44,8 +44,8 @@ public void testReadingDateFormats() throws IOException { verifySchema( result, schema("weekyear_week_day", null, "date"), - schema("hour_minute_second_millis", null, "time"), - schema("strict_ordinal_date_time", null, "timestamp")); + schema("hour_minute_second_millis", null, "date"), + schema("strict_ordinal_date_time", null, "date")); verifyDataRows(result, rows("1984-04-12", "09:07:42", "1984-04-12 09:07:42.000123456")); } @@ -68,11 +68,11 @@ public void testCustomFormats() { JSONObject result = executeQuery(query); verifySchema( result, - schema("custom_time", null, "time"), - schema("custom_timestamp", null, "timestamp"), + schema("custom_time", null, "date"), + schema("custom_timestamp", null, "date"), schema("custom_date_or_date", null, "date"), - schema("custom_date_or_custom_time", null, "timestamp"), - schema("custom_time_parser_check", null, "time")); + schema("custom_date_or_custom_time", null, "date"), + schema("custom_time_parser_check", null, "date")); verifyDataRows( result, rows( @@ -97,8 +97,8 @@ public void testCustomFormats2() { verifySchema( result, schema("custom_no_delimiter_date", null, "date"), - schema("custom_no_delimiter_time", null, "time"), - schema("custom_no_delimiter_ts", null, "timestamp")); + schema("custom_no_delimiter_time", null, "date"), + schema("custom_no_delimiter_ts", null, "date")); verifyDataRows( result, rows("1984-10-20", "10:20:30", "1984-10-20 15:35:48"), @@ -116,10 +116,10 @@ public void testIncompleteFormats() { JSONObject result = executeQuery(query); verifySchema( result, - schema("incomplete_1", null, "timestamp"), + schema("incomplete_1", null, "date"), schema("incomplete_2", null, "date"), - schema("incorrect", null, "timestamp"), - schema("incomplete_custom_time", null, "time"), + schema("incorrect", null, "date"), + schema("incomplete_custom_time", null, "date"), schema("incomplete_custom_date", null, "date")); verifyDataRows( result, @@ -133,8 +133,7 @@ public void testNumericFormats() { String query = String.format("SELECT epoch_sec, epoch_milli" + " FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); - verifySchema( - result, schema("epoch_sec", null, "timestamp"), schema("epoch_milli", null, "timestamp")); + verifySchema(result, schema("epoch_sec", null, "date"), schema("epoch_milli", null, "date")); verifyDataRows( result, rows("1970-01-01 00:00:42", "1970-01-01 00:00:00.042"), @@ -147,7 +146,7 @@ public void testDateNanosWithFormats() { String query = String.format("SELECT hour_minute_second_OR_t_time" + " FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("09:07:42"), rows("07:07:42.123456789")); } @@ -182,7 +181,7 @@ public void testDateNanosWithFunctions() { + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'", TEST_INDEX_DATE_FORMATS); result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("09:07:42")); query = String.format( @@ -190,7 +189,7 @@ public void testDateNanosWithFunctions() { + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'", TEST_INDEX_DATE_FORMATS); result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("07:07:42.123456789")); } @@ -203,7 +202,7 @@ public void testDateNanosOrderBy() { + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); - verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "date")); verifyDataRows(result, rows("07:07:42.123456789"), rows("09:07:42")); } @@ -225,7 +224,7 @@ public void testDateNanosWithNanos() { String query = String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC); JSONObject result = executeQuery(query); - verifySchema(result, schema("date_nanos_value", null, "timestamp")); + verifySchema(result, schema("date_nanos_value", null, "date")); verifyDataRows(result, rows("2019-03-24 01:34:46.123456789")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 0ec77f9f31..27e5796d17 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -1434,11 +1434,11 @@ public void testTimeBracket() throws IOException { public void testDateBracket() throws IOException { JSONObject result = executeQuery("select {date '2020-09-16'}"); verifySchema(result, schema("{date '2020-09-16'}", null, "date")); - verifyDataRows(result, rows("2020-09-16")); + verifyDataRows(result, rows("2020-09-16T00:00:00.000Z")); result = executeQuery("select {d '2020-09-16'}"); verifySchema(result, schema("{d '2020-09-16'}", null, "date")); - verifyDataRows(result, rows("2020-09-16")); + verifyDataRows(result, rows("2020-09-16T00:00:00.000Z")); } private void compareBrackets(String query1, String query2, String timestamp) throws IOException { @@ -1455,20 +1455,25 @@ public void testBracketedEquivalent() throws IOException { compareBrackets("timestamp", "ts", "2020-09-16 17:30:00"); compareBrackets("timestamp", "timestamp", "2020-09-16 17:30:00.123"); compareBrackets("timestamp", "ts", "2020-09-16 17:30:00.123"); - compareBrackets("date", "date", "2020-09-16"); - compareBrackets("date", "d", "2020-09-16"); + compareBrackets("date", "date", "2020-09-16T00:00:00.000Z"); + compareBrackets("date", "d", "2020-09-16T00:00:00.000Z"); compareBrackets("time", "time", "17:30:00"); compareBrackets("time", "t", "17:30:00"); } @Test - public void testBracketFails() { - assertThrows(ResponseException.class, () -> executeQuery("select {time '2020-09-16'}")); - assertThrows(ResponseException.class, () -> executeQuery("select {t '2020-09-16'}")); + public void testBrackets() { + try { + executeQuery("select {time '2020-09-16'}"); + executeQuery("select {t '2020-09-16'}"); + executeQuery("select {timestamp '2020-09-16'}"); + executeQuery("select {ts '2020-09-16'}"); + } catch (Exception e) { + // exception should not be thrown + assertTrue(false); + } assertThrows(ResponseException.class, () -> executeQuery("select {date '17:30:00'}")); assertThrows(ResponseException.class, () -> executeQuery("select {d '17:30:00'}")); - assertThrows(ResponseException.class, () -> executeQuery("select {timestamp '2020-09-16'}")); - assertThrows(ResponseException.class, () -> executeQuery("select {ts '2020-09-16'}")); assertThrows(ResponseException.class, () -> executeQuery("select {timestamp '17:30:00'}")); assertThrows(ResponseException.class, () -> executeQuery("select {ts '17:30:00'}")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java index f36992b1d0..4f8ee17de7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/JdbcFormatIT.java @@ -35,7 +35,7 @@ public void testSimpleDataTypesInSchema() { schema("account_number", "long"), schema("address", "text"), schema("age", "integer"), - schema("birthdate", "timestamp"), + schema("birthdate", "date"), schema("city", "keyword"), schema("male", "boolean"), schema("state", "text")); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 7129d058c0..cd70c5e020 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -65,15 +65,6 @@ public void typeof_opensearch_types() { TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, - rows( - "TEXT", - "TIMESTAMP", - "TIMESTAMP", - "BOOLEAN", - "OBJECT", - "KEYWORD", - "IP", - "BINARY", - "GEO_POINT")); + rows("TEXT", "DATE", "DATE", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index ddbba61260..7c9513afef 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -62,19 +62,23 @@ public String toString() { @EqualsAndHashCode.Exclude @Getter protected MappingType mappingType; // resolved ExprCoreType - protected ExprCoreType exprCoreType; + @Getter protected ExprCoreType exprCoreType; /** * Get a simplified type {@link ExprCoreType} if possible. To avoid returning `UNKNOWN` for - * `OpenSearch*Type`s, e.g. for IP, returns itself. + * `OpenSearch*Type`s, e.g. for IP, returns itself. If the `exprCoreType` is {@link + * ExprCoreType#DATE}, {@link ExprCoreType#TIMESTAMP}, {@link ExprCoreType#TIME}, or {@link + * ExprCoreType#UNKNOWN}, it returns the current instance; otherwise, it returns `exprCoreType`. * * @return An {@link ExprType}. */ public ExprType getExprType() { - if (exprCoreType != ExprCoreType.UNKNOWN) { - return exprCoreType; - } - return this; + return (exprCoreType == ExprCoreType.DATE + || exprCoreType == ExprCoreType.TIMESTAMP + || exprCoreType == ExprCoreType.TIME + || exprCoreType == ExprCoreType.UNKNOWN) + ? this + : exprCoreType; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 7e6bee77c2..3d7468b95a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -235,6 +235,17 @@ public List getAllCustomFormatters() { .collect(Collectors.toList()); } + /** + * Retrieves a list of custom formatters and OpenSearch named formatters defined by the user. + * + * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. + */ + public List getAllFormatters() { + List dateFormatters = this.getAllNamedFormatters(); + dateFormatters.addAll(this.getAllCustomFormatters()); + return dateFormatters; + } + /** * Retrieves a list of named formatters that format for dates. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 3341e01ab2..3cb182de5b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -230,7 +230,7 @@ private Optional type(String field) { private static ExprValue parseDateTimeString(String value, OpenSearchDateType dataType) { List formatters = dataType.getAllNamedFormatters(); formatters.addAll(dataType.getAllCustomFormatters()); - ExprCoreType returnFormat = (ExprCoreType) dataType.getExprType(); + ExprCoreType returnFormat = dataType.getExprCoreType(); for (DateFormatter formatter : formatters) { try { @@ -273,8 +273,7 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da private static ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; - ExprType returnFormat = dt.getExprType(); - + ExprCoreType returnFormat = dt.getExprCoreType(); if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); if (numFormatters.size() > 0 || !dt.hasFormats()) { @@ -287,7 +286,7 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) epochMillis = value.longValue(); } Instant instant = Instant.ofEpochMilli(epochMillis); - switch ((ExprCoreType) returnFormat) { + switch (returnFormat) { case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(ZoneOffset.UTC))); case DATE: diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index ff66ec425a..9ef2b77fac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -23,6 +23,7 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.span.SpanExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; /** Bucket Aggregation Builder. */ @@ -65,7 +66,11 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( .missingOrder(missingOrder) .order(sortOrder); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { + if (expr.getDelegated().type() instanceof OpenSearchDateType + && List.of(TIMESTAMP, TIME, DATE) + .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) { + sourceBuilder.userValuetypeHint(ValueType.LONG); + } else if (List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { sourceBuilder.userValuetypeHint(ValueType.LONG); } return helper.build(expr.getDelegated(), sourceBuilder::field, sourceBuilder::script); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 11533c754e..55a1e57c16 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -32,10 +32,13 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; /** Lucene query abstraction that builds Lucene query from function expression. */ public abstract class LuceneQuery { + private ReferenceExpression ref; + /** * Check if function expression supported by current Lucene query. Default behavior is that report * supported if: @@ -102,10 +105,11 @@ private boolean literalExpressionWrappedByCast(FunctionExpression func) { * @return query */ public QueryBuilder build(FunctionExpression func) { - ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); + this.ref = (ReferenceExpression) func.getArguments().get(0); Expression expr = func.getArguments().get(1); ExprValue literalValue = expr instanceof LiteralExpression ? expr.valueOf() : cast((FunctionExpression) expr); + return doBuild(ref.getAttr(), ref.type(), literalValue); } @@ -120,7 +124,7 @@ private ExprValue cast(FunctionExpression castFunction) { ImmutableMap.>builder() .put( BuiltinFunctionName.CAST_TO_STRING.getName(), - expr -> { + (expr) -> { if (!expr.type().equals(ExprCoreType.STRING)) { return new ExprStringValue(String.valueOf(expr.valueOf().value())); } else { @@ -209,7 +213,12 @@ private ExprValue cast(FunctionExpression castFunction) { .put( BuiltinFunctionName.CAST_TO_DATE.getName(), expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { + if (expr.type().equals(ExprCoreType.STRING) + && this.ref.type() instanceof OpenSearchDateType) { + return new ExprDateValue( + expr.valueOf().stringValue(), + ((OpenSearchDateType) this.ref.type()).getAllFormatters()); + } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprDateValue(expr.valueOf().stringValue()); } else { return new ExprDateValue(expr.valueOf().dateValue()); @@ -218,7 +227,12 @@ private ExprValue cast(FunctionExpression castFunction) { .put( BuiltinFunctionName.CAST_TO_TIME.getName(), expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { + if (expr.type().equals(ExprCoreType.STRING) + && this.ref.type() instanceof OpenSearchDateType) { + return new ExprTimeValue( + expr.valueOf().stringValue(), + ((OpenSearchDateType) this.ref.type()).getAllFormatters()); + } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprTimeValue(expr.valueOf().stringValue()); } else { return new ExprTimeValue(expr.valueOf().timeValue()); @@ -227,7 +241,12 @@ private ExprValue cast(FunctionExpression castFunction) { .put( BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), expr -> { - if (expr.type().equals(ExprCoreType.STRING)) { + if (expr.type().equals(ExprCoreType.STRING) + && this.ref.type() instanceof OpenSearchDateType) { + return new ExprTimestampValue( + expr.valueOf().stringValue(), + ((OpenSearchDateType) this.ref.type()).getAllFormatters()); + } else if (expr.type().equals(ExprCoreType.STRING)) { return new ExprTimestampValue(expr.valueOf().stringValue()); } else { return new ExprTimestampValue(expr.valueOf().timestampValue()); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java index 2e33e3cc7c..5ccc57f0ca 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/RangeQuery.java @@ -9,6 +9,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -30,6 +31,7 @@ public enum Comparison { @Override protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue literal) { + Object value = value(literal); RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName); @@ -48,7 +50,9 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l } private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + if (literal.type().equals(ExprCoreType.TIMESTAMP) + && ((ExprTimestampValue) literal).hasNoFormatter()) { + // Formatting to default Epoch when no custom formatter return literal.timestampValue().toEpochMilli(); } else { return literal.value(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java index cd506898d7..0b06658ea3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQuery.java @@ -7,6 +7,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -22,7 +23,9 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l } private Object value(ExprValue literal) { - if (literal.type().equals(ExprCoreType.TIMESTAMP)) { + if (literal.type().equals(ExprCoreType.TIMESTAMP) + && ((ExprTimestampValue) literal).hasNoFormatter()) { + // Formatting to default Epoch when no custom formatter return literal.timestampValue().toEpochMilli(); } else { return literal.value(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 82e6222dc4..3bbf8f19fb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -124,7 +124,15 @@ public void of_MappingType(MappingType mappingType, String name, ExprType dataTy assertAll( () -> assertEquals(nameForPPL, type.typeName()), () -> assertEquals(nameForSQL, type.legacyTypeName()), - () -> assertEquals(dataType, type.getExprType())); + () -> { + if (dataType == ExprCoreType.TIMESTAMP + || dataType == ExprCoreType.DATE + || dataType == ExprCoreType.TIME) { + assertEquals(dataType, type.getExprCoreType()); + } else { + assertEquals(dataType, type.getExprType()); + } + }); } @ParameterizedTest(name = "{0}") @@ -133,7 +141,7 @@ public void of_ExprCoreType(ExprCoreType coreType) { assumeFalse(coreType == UNKNOWN); var type = OpenSearchDataType.of(coreType); if (type instanceof OpenSearchDateType) { - assertEquals(coreType, type.getExprType()); + assertEquals(coreType, type.getExprCoreType()); } else { assertEquals(coreType.toString(), type.typeName()); assertEquals(coreType.toString(), type.legacyTypeName()); @@ -416,7 +424,7 @@ public void test_getExprType() { assertEquals(FLOAT, OpenSearchDataType.of(MappingType.HalfFloat).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.Double).getExprType()); assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); - assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); + assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprCoreType()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index c6885c8ffe..bd3e426077 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; @@ -48,8 +49,6 @@ class OpenSearchDateTypeTest { OpenSearchDateType.of(defaultFormatString); private static final OpenSearchDateType dateDateType = OpenSearchDateType.of(dateFormatString); private static final OpenSearchDateType timeDateType = OpenSearchDateType.of(timeFormatString); - private static final OpenSearchDateType datetimeDateType = - OpenSearchDateType.of(timestampFormatString); @Test public void isCompatible() { @@ -94,9 +93,9 @@ public void check_legacyTypeName() { public void check_exprTypeName() { assertAll( // exprType changes based on type (no datetime): - () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), - () -> assertEquals(TIME, timeDateType.getExprType()), - () -> assertEquals(DATE, dateDateType.getExprType())); + () -> assertEquals(TIMESTAMP, defaultDateType.getExprCoreType()), + () -> assertEquals(TIME, timeDateType.getExprCoreType()), + () -> assertEquals(DATE, dateDateType.getExprCoreType())); } private static Stream getAllSupportedFormats() { @@ -129,22 +128,22 @@ public void check_datetime_format_names(FormatNames datetimeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, camelCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } String snakeCaseName = datetimeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIMESTAMP, snakeCaseName + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); + + dateType.getExprCoreType()); } else { fail(); } @@ -161,18 +160,22 @@ public void check_date_format_names(FormatNames dateFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - camelCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = dateFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), DATE, - snakeCaseName + " does not format to a DATE type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a DATE type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -189,18 +192,22 @@ public void check_time_format_names(FormatNames timeFormat) { if (camelCaseName != null && !camelCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - camelCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + camelCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } String snakeCaseName = timeFormat.getSnakeCaseName(); if (snakeCaseName != null && !snakeCaseName.isEmpty()) { OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); assertSame( - dateType.getExprType(), + dateType.getExprCoreType(), TIME, - snakeCaseName + " does not format to a TIME type, instead got " + dateType.getExprType()); + snakeCaseName + + " does not format to a TIME type, instead got " + + dateType.getExprCoreType()); } else { fail(); } @@ -244,9 +251,9 @@ private static Stream get_format_combinations_for_test() { @MethodSource("get_format_combinations_for_test") public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( ExprCoreType expected, List formats, String testName) { - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); formats = Lists.reverse(formats); - assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprCoreType()); } @Test @@ -259,4 +266,16 @@ public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); assertFalse(isDateTypeCompatible(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); } + + @Test + public void testGetAllFormatters() { + List namedFormatters = timeDateType.getAllNamedFormatters(); + List customFormatters = timeDateType.getAllCustomFormatters(); + + List allFormatters = timeDateType.getAllFormatters(); + + assertEquals(namedFormatters.size() + customFormatters.size(), allFormatters.size()); + assertTrue(allFormatters.containsAll(namedFormatters)); + assertTrue(allFormatters.containsAll(customFormatters)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java index 3ddb07d86a..3ca566fac6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java @@ -148,7 +148,7 @@ void getFieldTypes() { hasEntry("gender", ExprCoreType.BOOLEAN), hasEntry("family", ExprCoreType.ARRAY), hasEntry("employer", ExprCoreType.STRUCT), - hasEntry("birthday", ExprCoreType.TIMESTAMP), + hasEntry("birthday", (ExprType) OpenSearchDataType.of(MappingType.Date)), hasEntry("id1", ExprCoreType.BYTE), hasEntry("id2", ExprCoreType.SHORT), hasEntry("blob", (ExprType) OpenSearchDataType.of(MappingType.Binary)))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 90b982e017..53b228622c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1767,7 +1767,7 @@ void cast_to_date_in_filter() { "{\n" + " \"term\" : {\n" + " \"date_value\" : {\n" - + " \"value\" : \"2021-11-08\",\n" + + " \"value\" : \"2021-11-08T00:00:00.000Z\",\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -1775,6 +1775,15 @@ void cast_to_date_in_filter() { assertJsonEquals( json, buildQuery(DSL.equal(ref("date_value", DATE), DSL.castDate(literal("2021-11-08"))))); + json = + "{\n" + + " \"term\" : {\n" + + " \"date_value\" : {\n" + + " \"value\" : \"2021-11-08\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}"; assertJsonEquals( json, buildQuery(