Skip to content

Commit

Permalink
Esql implicit casting for date nanos (elastic#118697)
Browse files Browse the repository at this point in the history 
resolves elastic#118476

This adds an implicit cast from string to date nanos, much the same as we do for millisecond dates. In the course of working on this, I found and fixed a couple of tests that were creating pre-epoch date nanos, which are not supported in elasticsearch. I also refactored the conversion code to use the standard DateUtils functions where appropriate, which caught some of the above errors in test data.
  • Loading branch information
@not-napoleon
not-napoleon authored Dec 17, 2024
1 parent 517abe4 commit ae3c0d7
Show file tree
Hide file tree
Showing 7 changed files with 179 additions and 22 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/118697.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 118697
summary: Esql implicit casting for date nanos
area: ES|QL
type: enhancement
issues:
- 118476
131 changes: 131 additions & 0 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,137 @@ millis:date | nanos:date_nanos | num:long
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000
;

implicit casting to nanos, date only
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) > "2023-10-23"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
;

implicit casting to nanos, date only, equality test
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) == "2023-10-23"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
;


implicit casting to nanos, date plus time to seconds
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
;

implicit casting to nanos, date plus time to seconds, equality test
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
;

implicit casting to nanos, date plus time to millis
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00.000"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
;

implicit casting to nanos, date plus time to millis, equality test
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28.948"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
;

implicit casting to nanos, date plus time to nanos
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) > "2023-10-23T00:00:00.000000000"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z
2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z
2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z
2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
;

implicit casting to nanos, date plus time to nanos, equality test
required_capability: date_nanos_type
required_capability: date_nanos_implicit_casting

FROM date_nanos
| WHERE MV_MIN(nanos) == "2023-10-23T12:27:28.948000000"
| SORT nanos DESC
| KEEP millis, nanos;

millis:date | nanos:date_nanos
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z
;

date nanos greater than millis
required_capability: date_nanos_type
required_capability: date_nanos_compare_to_millis
Expand Down
5 changes: 4 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,10 @@ public enum Cap {
* Support for mixed comparisons between nanosecond and millisecond dates
*/
DATE_NANOS_COMPARE_TO_MILLIS(),

/**
* Support implicit casting of strings to date nanos
*/
DATE_NANOS_IMPLICIT_CASTING(),
/**
* Support Least and Greatest functions on Date Nanos type
*/
Expand Down
31 changes: 17 additions & 14 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@
import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE;
import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
Expand Down Expand Up @@ -1050,21 +1051,23 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) {
/**
* Cast string literals in ScalarFunction, EsqlArithmeticOperation, BinaryComparison, In and GroupingFunction to desired data types.
* For example, the string literals in the following expressions will be cast implicitly to the field data type on the left hand side.
* date > "2024-08-21"
* date in ("2024-08-21", "2024-08-22", "2024-08-23")
* date = "2024-08-21" + 3 days
* ip == "127.0.0.1"
* version != "1.0"
* bucket(dateField, "1 month")
* date_trunc("1 minute", dateField)
*
* <ul>
* <li>date > "2024-08-21"</li>
* <li>date in ("2024-08-21", "2024-08-22", "2024-08-23")</li>
* <li>date = "2024-08-21" + 3 days</li>
* <li>ip == "127.0.0.1"</li>
* <li>version != "1.0"</li>
* <li>bucket(dateField, "1 month")</li>
* <li>date_trunc("1 minute", dateField)</li>
* </ul>
* If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if
* applicable. For example, implicit casting converts:
* Coalesce(Long, Int) to Coalesce(Long, Long)
* Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
* Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)
* Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)
*
* <ul>
* <li>Coalesce(Long, Int) to Coalesce(Long, Long)</li>
* <li>Coalesce(null, Long, Int) to Coalesce(null, Long, Long)</li>
* <li>Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)</li>
* <li>Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)</li>
* </ul>
* Coalesce(Int, Long) will NOT be converted to Coalesce(Long, Long) or Coalesce(Int, Int).
*/
private static class ImplicitCasting extends ParameterizedRule<LogicalPlan, LogicalPlan, AnalyzerContext> {
Expand Down Expand Up @@ -1245,7 +1248,7 @@ private static boolean supportsImplicitTemporalCasting(Expression e, BinaryOpera
}

private static boolean supportsStringImplicitCasting(DataType type) {
return type == DATETIME || type == IP || type == VERSION || type == BOOLEAN;
return type == DATETIME || type == DATE_NANOS || type == IP || type == VERSION || type == BOOLEAN;
}

private static UnresolvedAttribute unresolvedAttribute(Expression value, String type, Exception e) {
Expand Down
14 changes: 9 additions & 5 deletions ...ck/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import org.elasticsearch.common.logging.LoggerMessageFormat;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
Expand Down Expand Up @@ -51,7 +53,6 @@
import java.time.Period;
import java.time.ZoneId;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalAmount;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -200,6 +201,9 @@ public static Converter converterFor(DataType from, DataType to) {
if (to == DataType.DATETIME) {
return EsqlConverter.STRING_TO_DATETIME;
}
if (to == DATE_NANOS) {
return EsqlConverter.STRING_TO_DATE_NANOS;
}
if (to == DataType.IP) {
return EsqlConverter.STRING_TO_IP;
}
Expand Down Expand Up @@ -514,13 +518,12 @@ public static long dateTimeToLong(String dateTime, DateFormatter formatter) {
}

public static long dateNanosToLong(String dateNano) {
return dateNanosToLong(dateNano, DateFormatter.forPattern("strict_date_optional_time_nanos"));
return dateNanosToLong(dateNano, DEFAULT_DATE_NANOS_FORMATTER);
}

public static long dateNanosToLong(String dateNano, DateFormatter formatter) {
TemporalAccessor parsed = formatter.parse(dateNano);
long nanos = parsed.getLong(ChronoField.INSTANT_SECONDS) * 1_000_000_000 + parsed.getLong(ChronoField.NANO_OF_SECOND);
return nanos;
Instant parsed = DateFormatters.from(formatter.parse(dateNano)).toInstant();
return DateUtils.toLong(parsed);
}

public static String dateTimeToString(long dateTime) {
Expand Down Expand Up @@ -639,6 +642,7 @@ public enum EsqlConverter implements Converter {
STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataType.TIME_DURATION)),
STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono),
STRING_TO_DATETIME(x -> EsqlDataTypeConverter.dateTimeToLong((String) x)),
STRING_TO_DATE_NANOS(x -> EsqlDataTypeConverter.dateNanosToLong((String) x)),
STRING_TO_IP(x -> EsqlDataTypeConverter.stringToIP((String) x)),
STRING_TO_VERSION(x -> EsqlDataTypeConverter.stringToVersion((String) x)),
STRING_TO_DOUBLE(x -> EsqlDataTypeConverter.stringToDouble((String) x)),
Expand Down
2 changes: 1 addition & 1 deletion ...plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ private Page randomPage(List<ColumnInfoImpl> columns) {
case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(randomBoolean());
case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull();
// TODO - add a random instant thing here?
case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomLong());
case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomNonNegativeLong());
case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(new Version(randomIdentifier()).toBytesRef());
case GEO_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(GEO.asWkb(GeometryTestUtils.randomPoint()));
case CARTESIAN_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(CARTESIAN.asWkb(ShapeTestUtils.randomPoint()));
Expand Down
12 changes: 11 additions & 1 deletion ...ugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@

package org.elasticsearch.xpack.esql.type;

import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.esql.core.type.DataType;

import java.time.Instant;
import java.util.Arrays;
import java.util.List;

Expand Down Expand Up @@ -50,11 +52,19 @@
public class EsqlDataTypeConverterTests extends ESTestCase {

public void testNanoTimeToString() {
long expected = randomLong();
long expected = randomNonNegativeLong();
long actual = EsqlDataTypeConverter.dateNanosToLong(EsqlDataTypeConverter.nanoTimeToString(expected));
assertEquals(expected, actual);
}

public void testStringToDateNanos() {
assertEquals(
DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.000Z")),
EsqlDataTypeConverter.convert("2023-01-01T00:00:00.000000000", DATE_NANOS)
);
assertEquals(DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.000Z")), EsqlDataTypeConverter.convert("2023-01-01", DATE_NANOS));
}

public void testCommonTypeNull() {
for (DataType dataType : DataType.values()) {
assertEqualsCommonType(dataType, NULL, dataType);
Expand Down

0 comments on commit ae3c0d7

Please sign in to comment.