Skip to content

Commit

Permalink
Added New DateTime parser implementation (#11465)
Browse files Browse the repository at this point in the history 
Signed-off-by: Prabhat Sharma <[email protected]>
Co-authored-by: Prabhat Sharma <[email protected]>
(cherry picked from commit 829215c)
  • Loading branch information
@CaptainDredge
CaptainDredge authored and Prabhat Sharma committed Jan 31, 2024
1 parent 3758a64 commit 44aa53e
Show file tree
Hide file tree
Showing 10 changed files with 800 additions and 37 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Support index level allocation filtering for searchable snapshot index ([#11563](https://github.com/opensearch-project/OpenSearch/pull/11563))
- [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
- Add support for Google Application Default Credentials in repository-gcs ([#8394](https://github.com/opensearch-project/OpenSearch/pull/8394))
- New DateTime format for RFC3339 compatible date fields ([#11465](https://github.com/opensearch-project/OpenSearch/pull/11465))

### Dependencies
- Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
Expand Down Expand Up @@ -126,7 +127,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
- Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877)), ([#12000](https://github.com/opensearch-project/OpenSearch/pull/12000))
- Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968))
- Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))

### Deprecated

Expand Down
3 changes: 3 additions & 0 deletions NOTICE.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ Foundation (http://www.apache.org/).

This product includes software developed by
Joda.org (http://www.joda.org/).

This product includes software developed by
Morten Haraldsen (ethlo) (https://github.com/ethlo) under the Apache License, version 2.0.
37 changes: 37 additions & 0 deletions server/src/main/java/org/opensearch/common/time/DateFormatters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1299,6 +1299,41 @@ public class DateFormatters {
.withResolverStyle(ResolverStyle.STRICT)
);

/**
* Returns RFC 3339 a popular ISO 8601 profile compatible date time formatter and parser.
* This is not fully compatible to the existing spec, its more linient and closely follows w3c note on datetime
*/

public static final DateFormatter RFC3339_LENIENT_DATE_FORMATTER = new JavaDateFormatter(
"rfc3339_lenient",
new OpenSearchDateTimeFormatter(STRICT_DATE_OPTIONAL_TIME_PRINTER),
new RFC3339CompatibleDateTimeFormatter(
new DateTimeFormatterBuilder().append(DATE_FORMATTER)
.optionalStart()
.appendLiteral('T')
.appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
.appendLiteral(':')
.appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
.optionalStart()
.appendLiteral(':')
.appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
.optionalStart()
.appendFraction(NANO_OF_SECOND, 1, 9, true)
.optionalEnd()
.optionalStart()
.appendLiteral(',')
.appendFraction(NANO_OF_SECOND, 1, 9, false)
.optionalEnd()
.optionalStart()
.appendOffsetId()
.optionalEnd()
.optionalEnd()
.optionalEnd()
.toFormatter(Locale.ROOT)
.withResolverStyle(ResolverStyle.STRICT)
)
);

private static final DateTimeFormatter HOUR_MINUTE_SECOND_FORMATTER = new DateTimeFormatterBuilder().append(HOUR_MINUTE_FORMATTER)
.appendLiteral(":")
.appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
Expand Down Expand Up @@ -2152,6 +2187,8 @@ static DateFormatter forPattern(String input) {
return STRICT_YEAR_MONTH;
} else if (FormatNames.STRICT_YEAR_MONTH_DAY.matches(input)) {
return STRICT_YEAR_MONTH_DAY;
} else if (FormatNames.RFC3339_LENIENT.matches(input)) {
return RFC3339_LENIENT_DATE_FORMATTER;
} else {
try {
return new JavaDateFormatter(
Expand Down
1 change: 1 addition & 0 deletions server/src/main/java/org/opensearch/common/time/FormatNames.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
*/
public enum FormatNames {
ISO8601(null, "iso8601"),
RFC3339_LENIENT(null, "rfc3339_lenient"),
BASIC_DATE("basicDate", "basic_date"),
BASIC_DATE_TIME("basicDateTime", "basic_date_time"),
BASIC_DATE_TIME_NO_MILLIS("basicDateTimeNoMillis", "basic_date_time_no_millis"),
Expand Down
93 changes: 62 additions & 31 deletions server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.opensearch.core.common.Strings;

import java.text.ParsePosition;
import java.time.DateTimeException;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
Expand Down Expand Up @@ -69,11 +70,11 @@ class JavaDateFormatter implements DateFormatter {

private final String format;
private final String printFormat;
private final DateTimeFormatter printer;
private final List<DateTimeFormatter> parsers;
private final OpenSearchDateTimePrinter printer;
private final List<OpenSearchDateTimeFormatter> parsers;
private final JavaDateFormatter roundupParser;
private final Boolean canCacheLastParsedFormatter;
private volatile DateTimeFormatter lastParsedformatter = null;
private volatile OpenSearchDateTimeFormatter lastParsedformatter = null;

/**
* A round up formatter
Expand All @@ -82,11 +83,11 @@ class JavaDateFormatter implements DateFormatter {
*/
static class RoundUpFormatter extends JavaDateFormatter {

RoundUpFormatter(String format, List<DateTimeFormatter> roundUpParsers) {
RoundUpFormatter(String format, List<OpenSearchDateTimeFormatter> roundUpParsers) {
super(format, firstFrom(roundUpParsers), null, roundUpParsers);
}

private static DateTimeFormatter firstFrom(List<DateTimeFormatter> roundUpParsers) {
private static OpenSearchDateTimeFormatter firstFrom(List<OpenSearchDateTimeFormatter> roundUpParsers) {
return roundUpParsers.get(0);
}

Expand All @@ -100,14 +101,18 @@ JavaDateFormatter getRoundupParser() {
JavaDateFormatter(
String format,
String printFormat,
DateTimeFormatter printer,
OpenSearchDateTimePrinter printer,
Boolean canCacheLastParsedFormatter,
DateTimeFormatter... parsers
OpenSearchDateTimeFormatter... parsers
) {
this(format, printFormat, printer, ROUND_UP_BASE_FIELDS, canCacheLastParsedFormatter, parsers);
}

JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) {
this(format, format, wrapFormatter(printer), false, wrapAllFormatters(parsers));
}

JavaDateFormatter(String format, OpenSearchDateTimePrinter printer, OpenSearchDateTimeFormatter... parsers) {
this(format, format, printer, false, parsers);
}

Expand All @@ -126,19 +131,19 @@ JavaDateFormatter getRoundupParser() {
JavaDateFormatter(
String format,
String printFormat,
DateTimeFormatter printer,
OpenSearchDateTimePrinter printer,
BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
Boolean canCacheLastParsedFormatter,
DateTimeFormatter... parsers
OpenSearchDateTimeFormatter... parsers
) {
if (printer == null) {
throw new IllegalArgumentException("printer may not be null");
}
long distinctZones = Arrays.stream(parsers).map(DateTimeFormatter::getZone).distinct().count();
long distinctZones = Arrays.stream(parsers).map(OpenSearchDateTimeFormatter::getZone).distinct().count();
if (distinctZones > 1) {
throw new IllegalArgumentException("formatters must have the same time zone");
}
long distinctLocales = Arrays.stream(parsers).map(DateTimeFormatter::getLocale).distinct().count();
long distinctLocales = Arrays.stream(parsers).map(OpenSearchDateTimeFormatter::getLocale).distinct().count();
if (distinctLocales > 1) {
throw new IllegalArgumentException("formatters must have the same locale");
}
Expand All @@ -148,12 +153,12 @@ JavaDateFormatter getRoundupParser() {
this.canCacheLastParsedFormatter = canCacheLastParsedFormatter;

if (parsers.length == 0) {
this.parsers = Collections.singletonList(printer);
this.parsers = Collections.singletonList((OpenSearchDateTimeFormatter) printer);
} else {
this.parsers = Arrays.asList(parsers);
}
List<DateTimeFormatter> roundUp = createRoundUpParser(format, roundupParserConsumer);
this.roundupParser = new RoundUpFormatter(format, roundUp);
this.roundupParser = new RoundUpFormatter(format, wrapAllFormatters(roundUp));
}

JavaDateFormatter(
Expand All @@ -162,7 +167,7 @@ JavaDateFormatter getRoundupParser() {
BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
DateTimeFormatter... parsers
) {
this(format, format, printer, roundupParserConsumer, false, parsers);
this(format, format, wrapFormatter(printer), roundupParserConsumer, false, wrapAllFormatters(parsers));
}

/**
Expand All @@ -180,7 +185,8 @@ private List<DateTimeFormatter> createRoundUpParser(
) {
if (format.contains("||") == false) {
List<DateTimeFormatter> roundUpParsers = new ArrayList<>();
for (DateTimeFormatter parser : this.parsers) {
for (OpenSearchDateTimeFormatter customparser : this.parsers) {
DateTimeFormatter parser = customparser.getFormatter();
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
builder.append(parser);
roundupParserConsumer.accept(builder, parser);
Expand All @@ -200,12 +206,12 @@ public static DateFormatter combined(
assert formatters.size() > 0;
assert printFormatter != null;

List<DateTimeFormatter> parsers = new ArrayList<>(formatters.size());
List<DateTimeFormatter> roundUpParsers = new ArrayList<>(formatters.size());
List<OpenSearchDateTimeFormatter> parsers = new ArrayList<>(formatters.size());
List<OpenSearchDateTimeFormatter> roundUpParsers = new ArrayList<>(formatters.size());

assert printFormatter instanceof JavaDateFormatter;
JavaDateFormatter javaPrintFormatter = (JavaDateFormatter) printFormatter;
DateTimeFormatter printer = javaPrintFormatter.getPrinter();
OpenSearchDateTimePrinter printer = javaPrintFormatter.getPrinter();
for (DateFormatter formatter : formatters) {
assert formatter instanceof JavaDateFormatter;
JavaDateFormatter javaDateFormatter = (JavaDateFormatter) formatter;
Expand All @@ -226,9 +232,9 @@ public static DateFormatter combined(
private JavaDateFormatter(
String format,
String printFormat,
DateTimeFormatter printer,
List<DateTimeFormatter> roundUpParsers,
List<DateTimeFormatter> parsers,
OpenSearchDateTimePrinter printer,
List<OpenSearchDateTimeFormatter> roundUpParsers,
List<OpenSearchDateTimeFormatter> parsers,
Boolean canCacheLastParsedFormatter
) {
this.format = format;
Expand All @@ -244,6 +250,15 @@ private JavaDateFormatter(
DateTimeFormatter printer,
List<DateTimeFormatter> roundUpParsers,
List<DateTimeFormatter> parsers
) {
this(format, format, wrapFormatter(printer), wrapAllFormatters(roundUpParsers), wrapAllFormatters(parsers), false);
}

private JavaDateFormatter(
String format,
OpenSearchDateTimePrinter printer,
List<OpenSearchDateTimeFormatter> roundUpParsers,
List<OpenSearchDateTimeFormatter> parsers
) {
this(format, format, printer, roundUpParsers, parsers, false);
this.canCacheLastParsedFormatter = canCacheLastParsedFormatter;
Expand All @@ -262,7 +277,7 @@ JavaDateFormatter getRoundupParser() {
return roundupParser;
}

DateTimeFormatter getPrinter() {
OpenSearchDateTimePrinter getPrinter() {
return printer;
}

Expand All @@ -274,7 +289,7 @@ public TemporalAccessor parse(String input) {

try {
return doParse(input);
} catch (DateTimeParseException e) {
} catch (DateTimeException e) {
throw new IllegalArgumentException("failed to parse date field [" + input + "] with format [" + format + "]", e);
}
}
Expand All @@ -298,14 +313,14 @@ private TemporalAccessor doParse(String input) {
Object object = null;
if (canCacheLastParsedFormatter && lastParsedformatter != null) {
ParsePosition pos = new ParsePosition(0);
object = lastParsedformatter.toFormat().parseObject(input, pos);
object = lastParsedformatter.parseObject(input, pos);
if (parsingSucceeded(object, input, pos)) {
return (TemporalAccessor) object;
}
}
for (DateTimeFormatter formatter : parsers) {
for (OpenSearchDateTimeFormatter formatter : parsers) {
ParsePosition pos = new ParsePosition(0);
object = formatter.toFormat().parseObject(input, pos);
object = formatter.parseObject(input, pos);
if (parsingSucceeded(object, input, pos)) {
lastParsedformatter = formatter;
return (TemporalAccessor) object;
Expand All @@ -321,14 +336,28 @@ private boolean parsingSucceeded(Object object, String input, ParsePosition pos)
return object != null && pos.getIndex() == input.length();
}

private static OpenSearchDateTimeFormatter wrapFormatter(DateTimeFormatter formatter) {
return new OpenSearchDateTimeFormatter(formatter);
}

private static OpenSearchDateTimeFormatter[] wrapAllFormatters(DateTimeFormatter... formatters) {
return Arrays.stream(formatters).map(JavaDateFormatter::wrapFormatter).toArray(OpenSearchDateTimeFormatter[]::new);
}

private static List<OpenSearchDateTimeFormatter> wrapAllFormatters(List<DateTimeFormatter> formatters) {
return formatters.stream().map(JavaDateFormatter::wrapFormatter).collect(Collectors.toList());
}

@Override
public DateFormatter withZone(ZoneId zoneId) {
// shortcurt to not create new objects unnecessarily
if (zoneId.equals(zone())) {
return this;
}
List<DateTimeFormatter> parsers = this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList());
List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
List<OpenSearchDateTimeFormatter> parsers = new ArrayList<>(
this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList())
);
List<OpenSearchDateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
.stream()
.map(p -> p.withZone(zoneId))
.collect(Collectors.toList());
Expand All @@ -341,8 +370,10 @@ public DateFormatter withLocale(Locale locale) {
if (locale.equals(locale())) {
return this;
}
List<DateTimeFormatter> parsers = this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList());
List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
List<OpenSearchDateTimeFormatter> parsers = new ArrayList<>(
this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList())
);
List<OpenSearchDateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
.stream()
.map(p -> p.withLocale(locale))
.collect(Collectors.toList());
Expand Down Expand Up @@ -401,7 +432,7 @@ public String toString() {
return String.format(Locale.ROOT, "format[%s] locale[%s]", format, locale());
}

Collection<DateTimeFormatter> getParsers() {
Collection<OpenSearchDateTimeFormatter> getParsers() {
return parsers;
}
}
Loading

0 comments on commit 44aa53e

Please sign in to comment.