From ebf470acdf645b92dcc82e492699da5df3f6ab39 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 3 Dec 2024 13:04:02 -0800 Subject: [PATCH 1/4] Provide a mechanism to modify config files in a running test cluster (#117859) --- .../local/AbstractLocalClusterFactory.java | 103 +++++++++++++----- .../util/resource/MutableResource.java | 53 +++++++++ .../xpack/eql/EqlSecurityTestCluster.java | 2 +- 3 files changed, 128 insertions(+), 30 deletions(-) create mode 100644 test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/MutableResource.java diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 2dac2ee232aa5..6070ec140d254 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -26,6 +26,8 @@ import org.elasticsearch.test.cluster.util.ProcessUtils; import org.elasticsearch.test.cluster.util.Retry; import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.cluster.util.resource.MutableResource; +import org.elasticsearch.test.cluster.util.resource.Resource; import java.io.BufferedInputStream; import java.io.BufferedReader; @@ -115,6 +117,9 @@ public static class Node { private Version currentVersion; private Process process = null; private DistributionDescriptor distributionDescriptor; + private Set extraConfigListeners = new HashSet<>(); + private Set keystoreFileListeners = new HashSet<>(); + private Set roleFileListeners = new HashSet<>(); public Node(Path baseWorkingDir, DistributionResolver distributionResolver, LocalNodeSpec spec) { this(baseWorkingDir, distributionResolver, spec, null, false); @@ -436,6 +441,10 @@ private void writeConfiguration() { private void copyExtraConfigFiles() { spec.getExtraConfigFiles().forEach((fileName, resource) -> { + if (fileName.equals("roles.yml")) { + throw new IllegalArgumentException("Security roles should be configured via 'rolesFile()' method."); + } + final Path target = configDir.resolve(fileName); final Path directory = target.getParent(); if (Files.exists(directory) == false) { @@ -446,6 +455,14 @@ private void copyExtraConfigFiles() { } } resource.writeTo(target); + + // Register and update listener for this config file + if (resource instanceof MutableResource && extraConfigListeners.add(fileName)) { + ((MutableResource) resource).addUpdateListener(updated -> { + LOGGER.info("Updating config file '{}'", fileName); + updated.writeTo(target); + }); + } }); } @@ -485,29 +502,39 @@ private void addKeystoreSettings() { private void addKeystoreFiles() { spec.getKeystoreFiles().forEach((key, file) -> { - try { - Path path = Files.createTempFile(tempDir, key, null); - file.writeTo(path); - - ProcessUtils.exec( - spec.getKeystorePassword(), - workingDir, - OS.conditional( - c -> c.onWindows(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore.bat")) - .onUnix(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore")) - ), - getEnvironmentVariables(), - false, - "add-file", - key, - path.toString() - ).waitFor(); - } catch (InterruptedException | IOException e) { - throw new RuntimeException(e); + addKeystoreFile(key, file); + if (file instanceof MutableResource && keystoreFileListeners.add(key)) { + ((MutableResource) file).addUpdateListener(updated -> { + LOGGER.info("Updating keystore file '{}'", key); + addKeystoreFile(key, updated); + }); } }); } + private void addKeystoreFile(String key, Resource file) { + try { + Path path = Files.createTempFile(tempDir, key, null); + file.writeTo(path); + + ProcessUtils.exec( + spec.getKeystorePassword(), + workingDir, + OS.conditional( + c -> c.onWindows(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore.bat")) + .onUnix(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore")) + ), + getEnvironmentVariables(), + false, + "add-file", + key, + path.toString() + ).waitFor(); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + private void writeSecureSecretsFile() { if (spec.getKeystoreFiles().isEmpty() == false) { throw new IllegalStateException( @@ -535,16 +562,20 @@ private void configureSecurity() { if (spec.isSecurityEnabled()) { if (spec.getUsers().isEmpty() == false) { LOGGER.info("Setting up roles.yml for node '{}'", name); - - Path destination = workingDir.resolve("config").resolve("roles.yml"); - spec.getRolesFiles().forEach(rolesFile -> { - try ( - Writer writer = Files.newBufferedWriter(destination, StandardOpenOption.APPEND); - Reader reader = new BufferedReader(new InputStreamReader(rolesFile.asStream())) - ) { - reader.transferTo(writer); - } catch (IOException e) { - throw new UncheckedIOException("Failed to append roles file " + rolesFile + " to " + destination, e); + writeRolesFile(); + spec.getRolesFiles().forEach(resource -> { + if (resource instanceof MutableResource && roleFileListeners.add(resource)) { + ((MutableResource) resource).addUpdateListener(updated -> { + LOGGER.info("Updating roles.yml for node '{}'", name); + Path rolesFile = workingDir.resolve("config").resolve("roles.yml"); + try { + Files.delete(rolesFile); + Files.copy(distributionDir.resolve("config").resolve("roles.yml"), rolesFile); + writeRolesFile(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); } }); } @@ -596,6 +627,20 @@ private void configureSecurity() { } } + private void writeRolesFile() { + Path destination = workingDir.resolve("config").resolve("roles.yml"); + spec.getRolesFiles().forEach(rolesFile -> { + try ( + Writer writer = Files.newBufferedWriter(destination, StandardOpenOption.APPEND); + Reader reader = new BufferedReader(new InputStreamReader(rolesFile.asStream())) + ) { + reader.transferTo(writer); + } catch (IOException e) { + throw new UncheckedIOException("Failed to append roles file " + rolesFile + " to " + destination, e); + } + }); + } + private void installPlugins() { if (spec.getPlugins().isEmpty() == false) { Pattern pattern = Pattern.compile("(.+)(?:-\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?\\.zip)"); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/MutableResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/MutableResource.java new file mode 100644 index 0000000000000..477ad82e5944a --- /dev/null +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/MutableResource.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.test.cluster.util.resource; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +/** + * A mutable version of {@link Resource}. Anywhere a {@link Resource} is accepted in the test clusters API a {@link MutableResource} can + * be supplied instead. Unless otherwise specified, when the {@link #update(Resource)} method is called, the backing configuration will + * be updated in-place. + */ +public class MutableResource implements Resource { + private final List> listeners = new ArrayList<>(); + private Resource delegate; + + private MutableResource(Resource delegate) { + this.delegate = delegate; + } + + @Override + public InputStream asStream() { + return delegate.asStream(); + } + + public static MutableResource from(Resource delegate) { + return new MutableResource(delegate); + } + + public void update(Resource delegate) { + this.delegate = delegate; + this.listeners.forEach(listener -> listener.accept(this)); + } + + /** + * Registers a listener that will be notified when any updates are made to this resource. This listener will receive a reference to + * the resource with the updated value. + * + * @param listener action to be called on update + */ + public synchronized void addUpdateListener(Consumer listener) { + listeners.add(listener); + } +} diff --git a/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSecurityTestCluster.java b/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSecurityTestCluster.java index a1a417d91aeb8..33f048d81ef52 100644 --- a/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSecurityTestCluster.java +++ b/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSecurityTestCluster.java @@ -19,7 +19,7 @@ public static ElasticsearchCluster getCluster() { .setting("xpack.license.self_generated.type", "basic") .setting("xpack.monitoring.collection.enabled", "true") .setting("xpack.security.enabled", "true") - .configFile("roles.yml", Resource.fromClasspath("roles.yml")) + .rolesFile(Resource.fromClasspath("roles.yml")) .user("test-admin", "x-pack-test-password", "test-admin", false) .user("user1", "x-pack-test-password", "user1", false) .user("user2", "x-pack-test-password", "user2", false) From becf069ddad97bd4091d75ad1c0342c8d489a7f5 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 3 Dec 2024 16:13:46 -0500 Subject: [PATCH 2/4] Esql refactor date tests (#117923) This refactors a bit of the type logic in the parametrized testing to pass the input values as java Instants for millisecond and nanosecond date. Mainly, this impacts verifier functions. The goal here is to ensure that the values are correctly converted based on the type they were generated as, rather than relying on the verifier function to know how to convert from a long with no additional information. This will make tests that have mixed millisecond and nanosecond inputs easier to write correctly. --- .../xpack/esql/core/util/DateUtils.java | 4 + .../expression/function/TestCaseSupplier.java | 78 +++---------------- .../scalar/convert/ToDateNanosTests.java | 17 ++-- .../scalar/convert/ToDatetimeTests.java | 14 +++- .../scalar/convert/ToDoubleTests.java | 6 +- .../scalar/convert/ToIntegerTests.java | 7 +- .../function/scalar/convert/ToLongTests.java | 11 ++- .../scalar/convert/ToStringTests.java | 11 ++- .../scalar/convert/ToUnsignedLongTests.java | 6 +- .../operator/arithmetic/AddTests.java | 22 +++--- .../operator/arithmetic/SubTests.java | 16 ++-- .../comparison/GreaterThanOrEqualTests.java | 39 ++++------ .../operator/comparison/GreaterThanTests.java | 5 +- .../comparison/LessThanOrEqualTests.java | 39 ++++------ .../operator/comparison/LessThanTests.java | 5 +- 15 files changed, 117 insertions(+), 163 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java index 280cf172a8a58..20f7b400e9364 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java @@ -174,6 +174,10 @@ public static ZonedDateTime asDateTime(long millis) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC); } + public static ZonedDateTime asDateTime(Instant instant) { + return ZonedDateTime.ofInstant(instant, UTC); + } + public static long asMillis(ZonedDateTime zonedDateTime) { return zonedDateTime.toInstant().toEpochMilli(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 816c9ef6f352c..377027b70fb54 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -620,70 +620,6 @@ public static void forUnaryBoolean( unary(suppliers, expectedEvaluatorToString, booleanCases(), expectedType, v -> expectedValue.apply((Boolean) v), warnings); } - /** - * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. - * This variant defaults to maximum range of possible values - */ - public static void forUnaryDatetime( - List suppliers, - String expectedEvaluatorToString, - DataType expectedType, - Function expectedValue, - List warnings - ) { - unaryNumeric( - suppliers, - expectedEvaluatorToString, - dateCases(), - expectedType, - n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), - warnings - ); - } - - /** - * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. - * This variant accepts a range of values - */ - public static void forUnaryDatetime( - List suppliers, - String expectedEvaluatorToString, - DataType expectedType, - long min, - long max, - Function expectedValue, - List warnings - ) { - unaryNumeric( - suppliers, - expectedEvaluatorToString, - dateCases(min, max), - expectedType, - n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), - warnings - ); - } - - /** - * Generate positive test cases for a unary function operating on an {@link DataType#DATE_NANOS}. - */ - public static void forUnaryDateNanos( - List suppliers, - String expectedEvaluatorToString, - DataType expectedType, - Function expectedValue, - List warnings - ) { - unaryNumeric( - suppliers, - expectedEvaluatorToString, - dateNanosCases(), - expectedType, - n -> expectedValue.apply(DateUtils.toInstant((long) n)), - warnings - ); - } - /** * Generate positive test cases for a unary function operating on an {@link DataType#GEO_POINT}. */ @@ -1912,11 +1848,19 @@ public List multiRowData() { } /** - * @return the data value being supplied, casting unsigned longs into BigIntegers correctly + * @return the data value being supplied, casting to java objects when appropriate */ public Object getValue() { - if (type == DataType.UNSIGNED_LONG && data instanceof Long l) { - return NumericUtils.unsignedLongAsBigInteger(l); + if (data instanceof Long l) { + if (type == DataType.UNSIGNED_LONG) { + return NumericUtils.unsignedLongAsBigInteger(l); + } + if (type == DataType.DATETIME) { + return Instant.ofEpochMilli(l); + } + if (type == DataType.DATE_NANOS) { + return DateUtils.toInstant(l); + } } return data; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java index 485073d1a91d2..7459abf29410d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -36,14 +37,20 @@ public static Iterable parameters() { final String read = "Attribute[channel=0]"; final List suppliers = new ArrayList<>(); - TestCaseSupplier.forUnaryDateNanos(suppliers, read, DataType.DATE_NANOS, DateUtils::toLong, List.of()); - TestCaseSupplier.forUnaryDatetime( + TestCaseSupplier.unary( + suppliers, + read, + TestCaseSupplier.dateNanosCases(), + DataType.DATE_NANOS, + v -> DateUtils.toLong((Instant) v), + List.of() + ); + TestCaseSupplier.unary( suppliers, "ToDateNanosFromDatetimeEvaluator[field=" + read + "]", + TestCaseSupplier.dateCases(0, DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli()), DataType.DATE_NANOS, - 0, - DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli(), - i -> DateUtils.toNanoSeconds(i.toEpochMilli()), + i -> DateUtils.toNanoSeconds(((Instant) i).toEpochMilli()), List.of() ); TestCaseSupplier.forUnaryLong( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java index 2852b92ba156e..43b889baf5306 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java @@ -37,12 +37,20 @@ public static Iterable parameters() { final String read = "Attribute[channel=0]"; final List suppliers = new ArrayList<>(); - TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.DATETIME, Instant::toEpochMilli, emptyList()); - TestCaseSupplier.forUnaryDateNanos( + TestCaseSupplier.unary( + suppliers, + read, + TestCaseSupplier.dateCases(), + DataType.DATETIME, + v -> ((Instant) v).toEpochMilli(), + emptyList() + ); + TestCaseSupplier.unary( suppliers, "ToDatetimeFromDateNanosEvaluator[field=" + read + "]", + TestCaseSupplier.dateNanosCases(), DataType.DATETIME, - i -> DateUtils.toMilliSeconds(DateUtils.toLong(i)), + i -> DateUtils.toMilliSeconds(DateUtils.toLong((Instant) i)), emptyList() ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java index d5153019c1e41..b68306d6cac80 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -49,11 +50,12 @@ public static Iterable parameters() { ); TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataType.DOUBLE, b -> b ? 1d : 0d, List.of()); - TestCaseSupplier.forUnaryDatetime( + TestCaseSupplier.unary( suppliers, evaluatorName.apply("Long"), + TestCaseSupplier.dateCases(), DataType.DOUBLE, - i -> (double) i.toEpochMilli(), + i -> (double) ((Instant) i).toEpochMilli(), List.of() ); // random strings that don't look like a double diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java index eb81d48e0c5be..6a3f7022c9d3e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -48,7 +49,7 @@ public static Iterable parameters() { evaluatorName.apply("Long"), dateCases(0, Integer.MAX_VALUE), DataType.INTEGER, - l -> ((Long) l).intValue(), + l -> Long.valueOf(((Instant) l).toEpochMilli()).intValue(), List.of() ); // datetimes that fall outside Integer's range @@ -60,7 +61,9 @@ public static Iterable parameters() { l -> null, l -> List.of( "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", - "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + l + "] out of [integer] range" + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + + ((Instant) l).toEpochMilli() + + "] out of [integer] range" ) ); // random strings that don't look like an Integer diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index 4c2cf14af41e9..c7101ab730aba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -43,8 +43,15 @@ public static Iterable parameters() { TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataType.LONG, b -> b ? 1L : 0L, List.of()); // datetimes - TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.LONG, Instant::toEpochMilli, List.of()); - TestCaseSupplier.forUnaryDateNanos(suppliers, read, DataType.LONG, DateUtils::toLong, List.of()); + TestCaseSupplier.unary(suppliers, read, TestCaseSupplier.dateCases(), DataType.LONG, v -> ((Instant) v).toEpochMilli(), List.of()); + TestCaseSupplier.unary( + suppliers, + read, + TestCaseSupplier.dateNanosCases(), + DataType.LONG, + v -> DateUtils.toLong((Instant) v), + List.of() + ); // random strings that don't look like a long TestCaseSupplier.forUnaryStrings( suppliers, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java index 0b101efa073d9..3b30e4b353ae5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -81,18 +82,20 @@ public static Iterable parameters() { b -> new BytesRef(b.toString()), List.of() ); - TestCaseSupplier.forUnaryDatetime( + TestCaseSupplier.unary( suppliers, "ToStringFromDatetimeEvaluator[field=" + read + "]", + TestCaseSupplier.dateCases(), DataType.KEYWORD, - i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(i.toEpochMilli())), + i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(((Instant) i).toEpochMilli())), List.of() ); - TestCaseSupplier.forUnaryDateNanos( + TestCaseSupplier.unary( suppliers, "ToStringFromDateNanosEvaluator[field=" + read + "]", + TestCaseSupplier.dateNanosCases(), DataType.KEYWORD, - i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_NANOS_FORMATTER.formatNanos(DateUtils.toLong(i))), + i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_NANOS_FORMATTER.formatNanos(DateUtils.toLong((Instant) i))), List.of() ); TestCaseSupplier.forUnaryGeoPoint( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java index d8122aa73f81a..ca48bb029f223 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -58,11 +59,12 @@ public static Iterable parameters() { ); // datetimes - TestCaseSupplier.forUnaryDatetime( + TestCaseSupplier.unary( suppliers, evaluatorName.apply("Long"), + TestCaseSupplier.dateCases(), DataType.UNSIGNED_LONG, - instant -> BigInteger.valueOf(instant.toEpochMilli()), + instant -> BigInteger.valueOf(((Instant) instant).toEpochMilli()), List.of() ); // random strings that don't look like an unsigned_long diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index abfb634d5f301..aa4c037e5e961 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -159,7 +159,7 @@ public static Iterable parameters() { }; BiFunction> warnings = (lhs, rhs) -> { try { - addDatesAndTemporalAmount(lhs.data(), rhs.data(), AddTests::addMillis); + addDatesAndTemporalAmount(lhs.getValue(), rhs.getValue(), AddTests::addMillis); return List.of(); } catch (ArithmeticException e) { return List.of( @@ -193,6 +193,7 @@ public static Iterable parameters() { BinaryOperator nanosResult = (lhs, rhs) -> { try { + assert (lhs instanceof Instant) || (rhs instanceof Instant); return addDatesAndTemporalAmount(lhs, rhs, AddTests::addNanos); } catch (ArithmeticException e) { return null; @@ -327,29 +328,28 @@ private static String addErrorMessageString(boolean includeOrdinal, List adder) { + private static Object addDatesAndTemporalAmount(Object lhs, Object rhs, ToLongBiFunction adder) { // this weird casting dance makes the expected value lambda symmetric - Long date; + Instant date; TemporalAmount period; - if (lhs instanceof Long) { - date = (Long) lhs; + assert (lhs instanceof Instant) || (rhs instanceof Instant); + if (lhs instanceof Instant) { + date = (Instant) lhs; period = (TemporalAmount) rhs; } else { - date = (Long) rhs; + date = (Instant) rhs; period = (TemporalAmount) lhs; } return adder.applyAsLong(date, period); } - private static long addMillis(Long date, TemporalAmount period) { + private static long addMillis(Instant date, TemporalAmount period) { return asMillis(asDateTime(date).plus(period)); } - private static long addNanos(Long date, TemporalAmount period) { + private static long addNanos(Instant date, TemporalAmount period) { return DateUtils.toLong( - Instant.from( - ZonedDateTime.ofInstant(DateUtils.toInstant(date), org.elasticsearch.xpack.esql.core.util.DateUtils.UTC).plus(period) - ) + Instant.from(ZonedDateTime.ofInstant(date, org.elasticsearch.xpack.esql.core.util.DateUtils.UTC).plus(period)) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java index 1338299b3a121..bce5dea30f849 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java @@ -277,25 +277,23 @@ protected Expression build(Source source, List args) { return new Sub(source, args.get(0), args.get(1)); } - private static Object subtractDatesAndTemporalAmount(Object lhs, Object rhs, ToLongBiFunction subtract) { + private static Object subtractDatesAndTemporalAmount(Object lhs, Object rhs, ToLongBiFunction subtract) { // this weird casting dance makes the expected value lambda symmetric - Long date; + Instant date; TemporalAmount period; - if (lhs instanceof Long) { - date = (Long) lhs; + if (lhs instanceof Instant) { + date = (Instant) lhs; period = (TemporalAmount) rhs; } else { - date = (Long) rhs; + date = (Instant) rhs; period = (TemporalAmount) lhs; } return subtract.applyAsLong(date, period); } - private static long subtractNanos(Long date, TemporalAmount period) { + private static long subtractNanos(Instant date, TemporalAmount period) { return DateUtils.toLong( - Instant.from( - ZonedDateTime.ofInstant(DateUtils.toInstant(date), org.elasticsearch.xpack.esql.core.util.DateUtils.UTC).minus(period) - ) + Instant.from(ZonedDateTime.ofInstant(date, org.elasticsearch.xpack.esql.core.util.DateUtils.UTC).minus(period)) ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index a4f1a19e135ef..395a574028f6a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -106,33 +107,19 @@ public static Iterable parameters() { ) ); // Datetime - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "GreaterThanOrEqualLongsEvaluator", - "lhs", - "rhs", - (l, r) -> ((Number) l).longValue() >= ((Number) r).longValue(), - DataType.BOOLEAN, - TestCaseSupplier.dateCases(), - TestCaseSupplier.dateCases(), - List.of(), - false - ) - ); + suppliers.addAll(TestCaseSupplier.forBinaryNotCasting("GreaterThanOrEqualLongsEvaluator", "lhs", "rhs", (lhs, rhs) -> { + if (lhs instanceof Instant l && rhs instanceof Instant r) { + return l.isAfter(r) || l.equals(r); + } + throw new UnsupportedOperationException("Got some weird types"); + }, DataType.BOOLEAN, TestCaseSupplier.dateCases(), TestCaseSupplier.dateCases(), List.of(), false)); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "GreaterThanOrEqualLongsEvaluator", - "lhs", - "rhs", - (l, r) -> ((Number) l).longValue() >= ((Number) r).longValue(), - DataType.BOOLEAN, - TestCaseSupplier.dateNanosCases(), - TestCaseSupplier.dateNanosCases(), - List.of(), - false - ) - ); + suppliers.addAll(TestCaseSupplier.forBinaryNotCasting("GreaterThanOrEqualLongsEvaluator", "lhs", "rhs", (lhs, rhs) -> { + if (lhs instanceof Instant l && rhs instanceof Instant r) { + return l.isAfter(r) || l.equals(r); + } + throw new UnsupportedOperationException("Got some weird types"); + }, DataType.BOOLEAN, TestCaseSupplier.dateNanosCases(), TestCaseSupplier.dateNanosCases(), List.of(), false)); suppliers.addAll( TestCaseSupplier.stringCases( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index 86a4676e35009..b56ecd7392ba6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -111,7 +112,7 @@ public static Iterable parameters() { "GreaterThanLongsEvaluator", "lhs", "rhs", - (l, r) -> ((Number) l).longValue() > ((Number) r).longValue(), + (l, r) -> ((Instant) l).isAfter((Instant) r), DataType.BOOLEAN, TestCaseSupplier.dateCases(), TestCaseSupplier.dateCases(), @@ -125,7 +126,7 @@ public static Iterable parameters() { "GreaterThanLongsEvaluator", "lhs", "rhs", - (l, r) -> ((Number) l).longValue() > ((Number) r).longValue(), + (l, r) -> ((Instant) l).isAfter((Instant) r), DataType.BOOLEAN, TestCaseSupplier.dateNanosCases(), TestCaseSupplier.dateNanosCases(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 5793f26ecd447..60062f071c183 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -106,33 +107,19 @@ public static Iterable parameters() { ) ); // Datetime - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "LessThanOrEqualLongsEvaluator", - "lhs", - "rhs", - (l, r) -> ((Number) l).longValue() <= ((Number) r).longValue(), - DataType.BOOLEAN, - TestCaseSupplier.dateCases(), - TestCaseSupplier.dateCases(), - List.of(), - false - ) - ); + suppliers.addAll(TestCaseSupplier.forBinaryNotCasting("LessThanOrEqualLongsEvaluator", "lhs", "rhs", (lhs, rhs) -> { + if (lhs instanceof Instant l && rhs instanceof Instant r) { + return l.isBefore(r) || l.equals(r); + } + throw new UnsupportedOperationException("Got some weird types"); + }, DataType.BOOLEAN, TestCaseSupplier.dateCases(), TestCaseSupplier.dateCases(), List.of(), false)); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "LessThanOrEqualLongsEvaluator", - "lhs", - "rhs", - (l, r) -> ((Number) l).longValue() <= ((Number) r).longValue(), - DataType.BOOLEAN, - TestCaseSupplier.dateNanosCases(), - TestCaseSupplier.dateNanosCases(), - List.of(), - false - ) - ); + suppliers.addAll(TestCaseSupplier.forBinaryNotCasting("LessThanOrEqualLongsEvaluator", "lhs", "rhs", (lhs, rhs) -> { + if (lhs instanceof Instant l && rhs instanceof Instant r) { + return l.isBefore(r) || l.equals(r); + } + throw new UnsupportedOperationException("Got some weird types"); + }, DataType.BOOLEAN, TestCaseSupplier.dateNanosCases(), TestCaseSupplier.dateNanosCases(), List.of(), false)); suppliers.addAll( TestCaseSupplier.stringCases( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 0d114b4964920..30812cf8e538d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.math.BigInteger; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -111,7 +112,7 @@ public static Iterable parameters() { "LessThanLongsEvaluator", "lhs", "rhs", - (l, r) -> ((Number) l).longValue() < ((Number) r).longValue(), + (l, r) -> ((Instant) l).isBefore((Instant) r), DataType.BOOLEAN, TestCaseSupplier.dateNanosCases(), TestCaseSupplier.dateNanosCases(), @@ -125,7 +126,7 @@ public static Iterable parameters() { "LessThanLongsEvaluator", "lhs", "rhs", - (l, r) -> ((Number) l).longValue() < ((Number) r).longValue(), + (l, r) -> ((Instant) l).isBefore((Instant) r), DataType.BOOLEAN, TestCaseSupplier.dateCases(), TestCaseSupplier.dateCases(), From bf418cbcf94b1cb1063d1eba3cf864f8eb02f81e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 08:14:31 +1100 Subject: [PATCH 3/4] Mute org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests test20NoAutoGenerationWhenAutoConfigurationDisabled #117891 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7e5e7f15700f3..1259da6257dc3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -235,6 +235,9 @@ tests: - class: org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilderTests method: testToQuery issue: https://github.com/elastic/elasticsearch/issues/117904 +- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests + method: test20NoAutoGenerationWhenAutoConfigurationDisabled + issue: https://github.com/elastic/elasticsearch/issues/117891 # Examples: # From d582b82ae85f74150af374efa0f272378bb2d08a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 08:14:55 +1100 Subject: [PATCH 4/4] Mute org.elasticsearch.packaging.test.BootstrapCheckTests test20RunWithBootstrapChecks #117890 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1259da6257dc3..ec09f948fc3ef 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -238,6 +238,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test20NoAutoGenerationWhenAutoConfigurationDisabled issue: https://github.com/elastic/elasticsearch/issues/117891 +- class: org.elasticsearch.packaging.test.BootstrapCheckTests + method: test20RunWithBootstrapChecks + issue: https://github.com/elastic/elasticsearch/issues/117890 # Examples: #