Skip to content

Commit

Permalink
Add support for custom date format and openSearch date format for dat…
Browse files Browse the repository at this point in the history 
…e fields as part of Lucene query

Github Issue - opensearch-project#2700

Signed-off-by: Manasvini B S <[email protected]>
  • Loading branch information
@manasvinibs
manasvinibs committed Jul 11, 2024
1 parent d3ecb2d commit 13aa66d
Show file tree
Hide file tree
Showing 23 changed files with 611 additions and 95 deletions.
1 change: 1 addition & 0 deletions core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ dependencies {
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
api group: 'com.tdunning', name: 't-digest', version: '3.3'
api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
api project(':common')

testImplementation('org.junit.jupiter:junit-jupiter:5.9.3')
Expand Down
46 changes: 44 additions & 2 deletions core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,13 @@
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.time.DateFormatters;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.exception.SemanticCheckException;
Expand All @@ -25,6 +31,7 @@
public class ExprDateValue extends AbstractExprValue {

private final LocalDate date;
private String dateFormat;

/** Constructor of ExprDateValue. */
public ExprDateValue(String date) {
Expand All @@ -36,9 +43,44 @@ public ExprDateValue(String date) {
}
}

/** Constructor of ExprDateValue to support custom/OpenSearch date formats in mappings. */
public ExprDateValue(String date, List<DateFormatter> dateFormatters) {
LocalDate localDate = null;
String dateFormat = "";
// check if dateFormatters are empty, then set default ones
if (dateFormatters == null || dateFormatters.isEmpty()) {
List<String> defaultPatterns =
Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis");
dateFormatters = new ArrayList<>();
for (String pattern : defaultPatterns) {
dateFormatters.add(DateFormatter.forPattern(pattern));
}
}
for (DateFormatter formatter : dateFormatters) {
try {
TemporalAccessor accessor = formatter.parse(date);
ZonedDateTime zonedDateTime = DateFormatters.from(accessor);
localDate = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalDate();

dateFormat = formatter.format(accessor);
break;
} catch (IllegalArgumentException ignored) {
// nothing to do, try another format
}
}
if (localDate == null) {
localDate = new ExprDateValue(date).dateValue();
}
this.date = localDate;
this.dateFormat = dateFormat;
}

@Override
public String value() {
return DateTimeFormatter.ISO_LOCAL_DATE.format(date);
if (this.dateFormat == null || this.dateFormat.isEmpty()) {
return DateTimeFormatter.ISO_LOCAL_DATE.format(date);
}
return this.dateFormat;
}

@Override
Expand Down Expand Up @@ -68,7 +110,7 @@ public boolean isDateTime() {

@Override
public String toString() {
return String.format("DATE '%s'", value());
return String.format("DATE '%s'", DateTimeFormatter.ISO_LOCAL_DATE.format(date));
}

@Override
Expand Down
45 changes: 43 additions & 2 deletions core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,14 @@
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.time.DateFormatters;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.exception.SemanticCheckException;
Expand All @@ -26,6 +32,7 @@
public class ExprTimeValue extends AbstractExprValue {

private final LocalTime time;
private String timeFormat;

/** Constructor of ExprTimeValue. */
public ExprTimeValue(String time) {
Expand All @@ -37,9 +44,43 @@ public ExprTimeValue(String time) {
}
}

/** Constructor of ExprTimeValue to support custom/OpenSearch date formats in mappings. */
public ExprTimeValue(String time, List<DateFormatter> dateFormatters) {
LocalTime localTime = null;
String timeFormat = "";
// check if dateFormatters are empty, then set default ones
if (dateFormatters == null || dateFormatters.isEmpty()) {
List<String> defaultPatterns =
Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis");
dateFormatters = new ArrayList<>();
for (String pattern : defaultPatterns) {
dateFormatters.add(DateFormatter.forPattern(pattern));
}
}
for (DateFormatter formatter : dateFormatters) {
try {
TemporalAccessor accessor = formatter.parse(time);
ZonedDateTime zonedDateTime = DateFormatters.from(accessor);
localTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toLocalTime();
timeFormat = formatter.format(accessor);
break;
} catch (IllegalArgumentException ignored) {
// nothing to do, try another format
}
}
if (localTime == null) {
localTime = new ExprTimeValue(time).timeValue();
}
this.time = localTime;
this.timeFormat = timeFormat;
}

@Override
public String value() {
return ISO_LOCAL_TIME.format(time);
if (this.timeFormat == null || this.timeFormat.isEmpty()) {
return ISO_LOCAL_TIME.format(time);
}
return this.timeFormat;
}

@Override
Expand Down Expand Up @@ -67,7 +108,7 @@ public boolean isDateTime() {

@Override
public String toString() {
return String.format("TIME '%s'", value());
return String.format("TIME '%s'", ISO_LOCAL_TIME.format(time));
}

@Override
Expand Down
54 changes: 49 additions & 5 deletions core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,17 @@
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.time.DateFormatters;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.exception.SemanticCheckException;
Expand All @@ -26,6 +33,7 @@
public class ExprTimestampValue extends AbstractExprValue {

private final Instant timestamp;
private String dateTimeFormat;

/** Constructor. */
public ExprTimestampValue(String timestamp) {
Expand All @@ -42,18 +50,54 @@ public ExprTimestampValue(String timestamp) {
}
}

/**
* Constructor of ExprTimestampValue to support custom/OpenSearch dateTime formats in mappings.
*/
public ExprTimestampValue(String timestamp, List<DateFormatter> dateFormatters) {
Instant localDateTime = null;
String dateTimeFormat = "";
// check if dateFormatters are empty, then set default ones
if (dateFormatters == null || dateFormatters.isEmpty()) {
List<String> defaultPatterns =
Arrays.asList("strict_date_time_no_millis", "strict_date_optional_time", "epoch_millis");
dateFormatters = new ArrayList<>();
for (String pattern : defaultPatterns) {
dateFormatters.add(DateFormatter.forPattern(pattern));
}
}
for (DateFormatter formatter : dateFormatters) {
try {
TemporalAccessor accessor = formatter.parse(timestamp);
ZonedDateTime zonedDateTime = DateFormatters.from(accessor);
localDateTime = zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toInstant();
dateTimeFormat = formatter.format(accessor);
break;
} catch (IllegalArgumentException ignored) {
// nothing to do, try another format
}
}
if (localDateTime == null) {
localDateTime = new ExprTimestampValue(timestamp).timestampValue();
}
this.timestamp = localDateTime;
this.dateTimeFormat = dateTimeFormat;
}

/** localDateTime Constructor. */
public ExprTimestampValue(LocalDateTime localDateTime) {
this.timestamp = localDateTime.atZone(ZoneOffset.UTC).toInstant();
}

@Override
public String value() {
return timestamp.getNano() == 0
? DATE_TIME_FORMATTER_WITHOUT_NANO
.withZone(ZoneOffset.UTC)
.format(timestamp.truncatedTo(ChronoUnit.SECONDS))
: DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp);
if (this.dateTimeFormat == null || this.dateTimeFormat.isEmpty()) {
return timestamp.getNano() == 0
? DATE_TIME_FORMATTER_WITHOUT_NANO
.withZone(ZoneOffset.UTC)
.format(timestamp.truncatedTo(ChronoUnit.SECONDS))
: DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZoneOffset.UTC).format(timestamp);
}
return this.dateTimeFormat;
}

@Override
Expand Down
Loading

0 comments on commit 13aa66d

Please sign in to comment.