From a5526e5cc68f79b4e2e660b46f9909f3a0ffa75a Mon Sep 17 00:00:00 2001 From: Manasvini B S Date: Tue, 18 Jun 2024 12:25:40 -0700 Subject: [PATCH] Fix to support different date formats in the sql query without date casting Github Issue - https://github.com/opensearch-project/sql/issues/2700 Signed-off-by: Manasvini B S --- .../sql/data/model/ExprDateValue.java | 38 ++++++++++++++++++- .../sql/utils/DateTimeFormatters.java | 14 ++++++- .../sql/data/model/DateTimeValueTest.java | 29 ++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index c36cd3ea6d..1242eeedbc 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -5,16 +5,22 @@ package org.opensearch.sql.data.model; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_PATTERNS_NANOS_OPTIONAL; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL; import com.google.common.base.Objects; + import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; +import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.format.ResolverStyle; +import java.util.Locale; + import lombok.RequiredArgsConstructor; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -25,10 +31,12 @@ public class ExprDateValue extends AbstractExprValue { private final LocalDate date; + private String datePattern; /** Constructor of ExprDateValue. */ public ExprDateValue(String date) { try { + this.datePattern = determineDatePattern(date); this.date = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException( @@ -36,9 +44,35 @@ public ExprDateValue(String date) { } } + private String determineDatePattern(String date) { + for (String pattern : DATE_TIME_FORMATTER_PATTERNS_NANOS_OPTIONAL) { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern, Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); + if (pattern.contains("HH") || pattern.contains("mm") || pattern.contains("ss")) { + LocalDateTime.parse(date, formatter); + } else { + LocalDate.parse(date, formatter); + } + return pattern; + } catch (DateTimeParseException e) { + // Ignore and try next pattern + } + } + return null; + } + @Override public String value() { - return DateTimeFormatter.ISO_LOCAL_DATE.format(date); + if (this.datePattern == null) { + return DateTimeFormatter.ISO_LOCAL_DATE.format(date); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(this.datePattern, Locale.ROOT); + if (this.datePattern.contains("HH") || this.datePattern.contains("mm") || this.datePattern.contains("ss")) { + LocalDateTime dateValueWithDefaultTime = this.date.atTime(0, 0, 0); + return dateValueWithDefaultTime.format(formatter); + } + return this.date.format(formatter); } @Override @@ -68,7 +102,7 @@ public boolean isDateTime() { @Override public String toString() { - return String.format("DATE '%s'", value()); + return String.format("DATE '%s'", DateTimeFormatter.ISO_LOCAL_DATE.format(date)); } @Override diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 18e6541514..6bf2d55c58 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -18,6 +18,8 @@ import java.time.format.ResolverStyle; import java.time.format.SignStyle; import java.time.temporal.ChronoField; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import lombok.experimental.UtilityClass; @@ -43,6 +45,16 @@ public class DateTimeFormatters { private static final int MIN_FRACTION_SECONDS = 0; private static final int MAX_FRACTION_SECONDS = 9; + // Define the list of date time patterns + public static final List DATE_TIME_FORMATTER_PATTERNS_NANOS_OPTIONAL = Arrays.asList( + "uuuu-MM-dd HH:mm:ss", + "uuuu-MM-dd HH:mm", + "HH:mm:ss", + "HH:mm", + "uuuu-MM-dd", + "dd-MMM-uu" + ); + public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() .appendOffset("+HHmm", "Z") @@ -130,7 +142,7 @@ public class DateTimeFormatters { public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL = new DateTimeFormatterBuilder() - .appendPattern("[uuuu-MM-dd HH:mm:ss][uuuu-MM-dd HH:mm][HH:mm:ss][HH:mm][uuuu-MM-dd]") + .appendPattern("[" + String.join("][", DATE_TIME_FORMATTER_PATTERNS_NANOS_OPTIONAL) + "]") .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) .toFormatter(Locale.ROOT) diff --git a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java index b5a3d61211..d06fb905e4 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java @@ -95,12 +95,41 @@ public void dateValueInterfaceTest() { assertEquals("invalid to get dateValue from value of type INTEGER", exception.getMessage()); } + @Test + public void dateValueWithSupportedDateTimeFormatTest() { + ExprDateValue dateTimeWithSecValue = new ExprDateValue("2020-08-17 12:12:00"); + assertEquals("2020-08-17 00:00:00", dateTimeWithSecValue.value()); + assertEquals(LocalDate.of(2020, 8, 17), dateTimeWithSecValue.dateValue()); + + ExprDateValue dateTimeWithoutSecValue = new ExprDateValue("2020-08-17 12:12"); + assertEquals("2020-08-17 00:00", dateTimeWithoutSecValue.value()); + assertEquals(LocalDate.of(2020, 8, 17), dateTimeWithoutSecValue.dateValue()); + + ExprDateValue dateValue = new ExprDateValue("2020-08-17"); + assertEquals("2020-08-17", dateValue.value()); + assertEquals(LocalDate.of(2020, 8, 17), dateValue.dateValue()); + + ExprDateValue dateValue2 = new ExprDateValue("03-Jan-21"); + assertEquals("03-Jan-21", dateValue2.value()); + assertEquals(LocalDate.of(2021, 1, 3), dateValue2.dateValue()); + } + @Test public void dateInUnsupportedFormat() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> new ExprDateValue("2020-07-07Z")); assertEquals( "date:2020-07-07Z in unsupported format, please use 'yyyy-MM-dd'", exception.getMessage()); + + SemanticCheckException timeException = + assertThrows(SemanticCheckException.class, () -> new ExprDateValue("01:01:01")); + assertEquals( + "date:01:01:01 in unsupported format, please use 'yyyy-MM-dd'", timeException.getMessage()); + + SemanticCheckException timeWithoutSecException = + assertThrows(SemanticCheckException.class, () -> new ExprDateValue("01:45")); + assertEquals( + "date:01:45 in unsupported format, please use 'yyyy-MM-dd'", timeWithoutSecException.getMessage()); } @Test