From b0173564ff96dc19e77bd3c8ee8dff600ddab771 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:44:42 +0200 Subject: [PATCH 01/23] Doc parsing error logging with throttling (#117828) * Throttled doc parsing error logging * add test * move throttler to separate class * small changes * refactor unittest * fix test --- .../index/mapper/DocumentMapper.java | 10 +-- .../index/mapper/IntervalThrottler.java | 66 +++++++++++++++++++ .../index/mapper/DocumentMapperTests.java | 37 +++++++++++ .../index/mapper/IntervalThrottlerTests.java | 27 ++++++++ 4 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IntervalThrottler.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IntervalThrottlerTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 1c9321737ab5f..c56885eded38f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -21,6 +21,8 @@ import java.util.List; public class DocumentMapper { + static final NodeFeature INDEX_SORTING_ON_NESTED = new NodeFeature("mapper.index_sorting_on_nested"); + private final String type; private final CompressedXContent mappingSource; private final MappingLookup mappingLookup; @@ -29,8 +31,6 @@ public class DocumentMapper { private final IndexVersion indexVersion; private final Logger logger; - static final NodeFeature INDEX_SORTING_ON_NESTED = new NodeFeature("mapper.index_sorting_on_nested"); - /** * Create a new {@link DocumentMapper} that holds empty mappings. * @param mapperService the mapper service that holds the needed components @@ -72,9 +72,11 @@ public static DocumentMapper createEmpty(MapperService mapperService) { : "provided source [" + source + "] differs from mapping [" + mapping.toCompressedXContent() + "]"; } - private void maybeLogDebug(Exception ex) { + private void maybeLog(Exception ex) { if (logger.isDebugEnabled()) { logger.debug("Error while parsing document: " + ex.getMessage(), ex); + } else if (IntervalThrottler.DOCUMENT_PARSING_FAILURE.accept()) { + logger.error("Error while parsing document: " + ex.getMessage(), ex); } } @@ -125,7 +127,7 @@ public ParsedDocument parse(SourceToParse source) throws DocumentParsingExceptio try { return documentParser.parseDocument(source, mappingLookup); } catch (Exception e) { - maybeLogDebug(e); + maybeLog(e); throw e; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IntervalThrottler.java b/server/src/main/java/org/elasticsearch/index/mapper/IntervalThrottler.java new file mode 100644 index 0000000000000..ffc35d9eaf769 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IntervalThrottler.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Throttles tracked operations based on a time interval, restricting them to 1 per N seconds. + */ +enum IntervalThrottler { + DOCUMENT_PARSING_FAILURE(60); + + static final int MILLISECONDS_IN_SECOND = 1000; + + private final Acceptor acceptor; + + IntervalThrottler(long intervalSeconds) { + acceptor = new Acceptor(intervalSeconds * MILLISECONDS_IN_SECOND); + } + + /** + * @return true if the operation gets accepted, false if throttled. + */ + boolean accept() { + return acceptor.accept(); + } + + // Defined separately for testing. + static class Acceptor { + private final long intervalMillis; + private final AtomicBoolean lastAcceptedGuard = new AtomicBoolean(false); + private volatile long lastAcceptedTimeMillis = 0; + + Acceptor(long intervalMillis) { + this.intervalMillis = intervalMillis; + } + + boolean accept() { + final long now = System.currentTimeMillis(); + // Check without guarding first, to reduce contention. + if (now - lastAcceptedTimeMillis > intervalMillis) { + // Check if another concurrent operation succeeded. + if (lastAcceptedGuard.compareAndSet(false, true)) { + try { + // Repeat check under guard protection, so that only one message gets written per interval. + if (now - lastAcceptedTimeMillis > intervalMillis) { + lastAcceptedTimeMillis = now; + return true; + } + } finally { + // Reset guard. + lastAcceptedGuard.set(false); + } + } + } + return false; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index db9fdead949de..b2ba3d60d2174 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -9,12 +9,18 @@ package org.elasticsearch.index.mapper; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LogEvent; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.MockAppender; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -493,4 +499,35 @@ public void testDeeplyNestedMapping() throws Exception { } } } + + public void testParsingErrorLogging() throws Exception { + MockAppender appender = new MockAppender("mock_appender"); + appender.start(); + Logger testLogger = LogManager.getLogger(DocumentMapper.class); + Loggers.addAppender(testLogger, appender); + Level originalLogLevel = testLogger.getLevel(); + Loggers.setLevel(testLogger, Level.ERROR); + + try { + DocumentMapper doc = createDocumentMapper(mapping(b -> b.startObject("value").field("type", "integer").endObject())); + + DocumentParsingException e = expectThrows( + DocumentParsingException.class, + () -> doc.parse(source(b -> b.field("value", "foo"))) + ); + assertThat(e.getMessage(), containsString("failed to parse field [value] of type [integer] in document with id '1'")); + LogEvent event = appender.getLastEventAndReset(); + if (event != null) { + assertThat(event.getMessage().getFormattedMessage(), containsString(e.getMessage())); + } + + e = expectThrows(DocumentParsingException.class, () -> doc.parse(source(b -> b.field("value", "foo")))); + assertThat(e.getMessage(), containsString("failed to parse field [value] of type [integer] in document with id '1'")); + assertThat(appender.getLastEventAndReset(), nullValue()); + } finally { + Loggers.setLevel(testLogger, originalLogLevel); + Loggers.removeAppender(testLogger, appender); + appender.stop(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntervalThrottlerTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntervalThrottlerTests.java new file mode 100644 index 0000000000000..25fd614524441 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IntervalThrottlerTests.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.test.ESTestCase; + +public class IntervalThrottlerTests extends ESTestCase { + + public void testThrottling() throws Exception { + var throttler = new IntervalThrottler.Acceptor(10); + assertTrue(throttler.accept()); + assertFalse(throttler.accept()); + assertFalse(throttler.accept()); + + Thread.sleep(20); + assertTrue(throttler.accept()); + assertFalse(throttler.accept()); + assertFalse(throttler.accept()); + } +} From cda2fe68a3a549e2443ca7137b4df431af6f2ba7 Mon Sep 17 00:00:00 2001 From: Luke Whiting Date: Wed, 4 Dec 2024 10:55:13 +0000 Subject: [PATCH 02/23] [CI] DocsClientYamlTestSuiteIT test {yaml=reference/watcher/example-watches/example-watch-clusterstatus/line_137} failing - (#115809) (#117354) * Ignore system index access errors in YAML test index cleanup method * Remove test mute * Swap the logic back as it was right the first time * Resolve conflict with latest merge * Move warning handler into it's own method to reduce nesting --- muted-tests.yml | 3 -- .../test/rest/ESRestTestCase.java | 41 +++++++++---------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index b2f5b08319ff7..17a7b26d1c091 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -100,9 +100,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_start_stop/Verify start transform reuses destination index} issue: https://github.com/elastic/elasticsearch/issues/115808 -- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT - method: test {yaml=reference/watcher/example-watches/example-watch-clusterstatus/line_137} - issue: https://github.com/elastic/elasticsearch/issues/115809 - class: org.elasticsearch.search.StressSearchServiceReaperIT method: testStressReaper issue: https://github.com/elastic/elasticsearch/issues/115816 diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index bdef0ba631b72..b4f4243fb90fd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -26,6 +26,8 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -158,6 +160,8 @@ public abstract class ESRestTestCase extends ESTestCase { private static final Pattern SEMANTIC_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*"); + private static final Logger SUITE_LOGGER = LogManager.getLogger(ESRestTestCase.class); + /** * Convert the entity from a {@link Response} into a map of maps. * Consumes the underlying HttpEntity, releasing any resources it may be holding. @@ -1111,7 +1115,14 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE } final Request deleteRequest = new Request("DELETE", Strings.collectionToCommaDelimitedString(indexPatterns)); deleteRequest.addParameter("expand_wildcards", "open,closed,hidden"); - deleteRequest.setOptions(deleteRequest.getOptions().toBuilder().setWarningsHandler(ignoreAsyncSearchWarning()).build()); + + // If system index warning, ignore but log + // See: https://github.com/elastic/elasticsearch/issues/117099 + // and: https://github.com/elastic/elasticsearch/issues/115809 + deleteRequest.setOptions( + RequestOptions.DEFAULT.toBuilder().setWarningsHandler(ESRestTestCase::ignoreSystemIndexAccessWarnings) + ); + final Response response = adminClient().performRequest(deleteRequest); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); @@ -1124,28 +1135,16 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE } } - // Make warnings handler that ignores the .async-search warning since .async-search may randomly appear when async requests are slow - // See: https://github.com/elastic/elasticsearch/issues/117099 - protected static WarningsHandler ignoreAsyncSearchWarning() { - return new WarningsHandler() { - @Override - public boolean warningsShouldFailRequest(List warnings) { - if (warnings.isEmpty()) { - return false; - } - return warnings.equals( - List.of( - "this request accesses system indices: [.async-search], " - + "but in a future major version, direct access to system indices will be prevented by default" - ) - ) == false; + private static boolean ignoreSystemIndexAccessWarnings(List warnings) { + for (String warning : warnings) { + if (warning.startsWith("this request accesses system indices:")) { + SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning); + } else { + return true; } + } - @Override - public String toString() { - return "ignore .async-search warning"; - } - }; + return false; } protected static void wipeDataStreams() throws IOException { From 4496aa0cbf5b23bc3146d4dd09d2b6194e3c9e5f Mon Sep 17 00:00:00 2001 From: Jakob Reiter Date: Wed, 4 Dec 2024 11:56:23 +0100 Subject: [PATCH 03/23] Update troubleshooting-unstable-cluster.asciidoc (#117887) Added missing word --- .../troubleshooting/troubleshooting-unstable-cluster.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc index cbb35f7731034..e47b85aa99547 100644 --- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc @@ -126,7 +126,7 @@ repeatedly-dropped connections will severely affect its operation. The connections from the elected master node to every other node in the cluster are particularly important. The elected master never spontaneously closes its outbound connections to other nodes. Similarly, once an inbound connection is -fully established, a node never spontaneously it unless the node is shutting +fully established, a node never spontaneously closes it unless the node is shutting down. If you see a node unexpectedly leave the cluster with the `disconnected` From 032b42fcf7c398391d8197c5dcdbf7ee20628ce7 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:17:13 +0100 Subject: [PATCH 04/23] Make TransportLocalClusterStateAction wait for cluster to unblock (#117230) This will make `TransportLocalClusterStateAction` wait for a new state that is not blocked. This means we need a timeout (again). For consistency's sake, we're reusing the REST param `master_timeout` for this timeout as well. The only class that was using `TransportLocalClusterStateAction` was `TransportGetAliasesAction`, so its request needed to accept a timeout again as well. --- docs/changelog/117230.yaml | 5 + docs/reference/cat/alias.asciidoc | 2 + docs/reference/indices/alias-exists.asciidoc | 2 + docs/reference/indices/get-alias.asciidoc | 2 + .../rest-api-spec/api/cat.aliases.json | 5 + .../api/indices.exists_alias.json | 5 + .../rest-api-spec/api/indices.get_alias.json | 5 + server/src/main/java/module-info.java | 1 + .../indices/alias/get/GetAliasesRequest.java | 21 ++- .../alias/get/TransportGetAliasesAction.java | 2 +- .../TransportLocalClusterStateAction.java | 64 ------- .../local/LocalClusterStateRequest.java | 43 +++++ .../TransportLocalClusterStateAction.java | 128 +++++++++++++ .../admin/indices/RestGetAliasesAction.java | 4 +- .../rest/action/cat/RestAliasAction.java | 9 +- ...TransportLocalClusterStateActionTests.java | 177 ++++++++++++++++++ 16 files changed, 396 insertions(+), 79 deletions(-) create mode 100644 docs/changelog/117230.yaml delete mode 100644 server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java create mode 100644 server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java create mode 100644 server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java diff --git a/docs/changelog/117230.yaml b/docs/changelog/117230.yaml new file mode 100644 index 0000000000000..001dcef2fe3b1 --- /dev/null +++ b/docs/changelog/117230.yaml @@ -0,0 +1,5 @@ +pr: 117230 +summary: Make various alias retrieval APIs wait for cluster to unblock +area: Distributed +type: enhancement +issues: [] diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index 41ac279d3b2f5..aab0c9df25ed4 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -51,6 +51,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + [[cat-alias-api-example]] ==== {api-examples-title} diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc index d7b3454dcff56..a514d36a1bfef 100644 --- a/docs/reference/indices/alias-exists.asciidoc +++ b/docs/reference/indices/alias-exists.asciidoc @@ -52,6 +52,8 @@ Defaults to `all`. (Optional, Boolean) If `false`, requests that include a missing data stream or index in the `` return an error. Defaults to `false`. +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + [[alias-exists-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc index 41d62fb70e01b..d4c5b92116949 100644 --- a/docs/reference/indices/get-alias.asciidoc +++ b/docs/reference/indices/get-alias.asciidoc @@ -58,3 +58,5 @@ Defaults to `all`. `ignore_unavailable`:: (Optional, Boolean) If `false`, requests that include a missing data stream or index in the `` return an error. Defaults to `false`. + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json index d3856b455efd1..9f97fe6280dc0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json @@ -65,6 +65,11 @@ ], "default": "all", "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "master_timeout":{ + "type":"time", + "description":"Timeout for waiting for new cluster state in case it is blocked", + "default":"30s" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json index 7d7a9c96c6419..7c855335efd00 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json @@ -61,6 +61,11 @@ ], "default":"all", "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "master_timeout":{ + "type":"time", + "description":"Timeout for waiting for new cluster state in case it is blocked", + "default":"30s" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json index dc02a65adb068..a360582a44a04 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json @@ -79,6 +79,11 @@ ], "default": "all", "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "master_timeout":{ + "type":"time", + "description":"Timeout for waiting for new cluster state in case it is blocked", + "default":"30s" } } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 5acc202ebb294..331a2bc0dddac 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -147,6 +147,7 @@ exports org.elasticsearch.action.support.master; exports org.elasticsearch.action.support.master.info; exports org.elasticsearch.action.support.nodes; + exports org.elasticsearch.action.support.local; exports org.elasticsearch.action.support.replication; exports org.elasticsearch.action.support.single.instance; exports org.elasticsearch.action.support.single.shard; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index c28108815ed03..af95787a5fcca 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -8,21 +8,20 @@ */ package org.elasticsearch.action.admin.indices.alias.get; -import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.AliasesRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; +import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import java.io.IOException; import java.util.Map; -public class GetAliasesRequest extends ActionRequest implements AliasesRequest { +public class GetAliasesRequest extends LocalClusterStateRequest implements AliasesRequest { public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandHidden(); @@ -31,18 +30,20 @@ public class GetAliasesRequest extends ActionRequest implements AliasesRequest { private String[] indices = Strings.EMPTY_ARRAY; private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + @Deprecated public GetAliasesRequest(String... aliases) { - this.aliases = aliases; - this.originalAliases = aliases; + this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, aliases); } + @Deprecated public GetAliasesRequest() { this(Strings.EMPTY_ARRAY); } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); + public GetAliasesRequest(TimeValue masterTimeout, String... aliases) { + super(masterTimeout); + this.aliases = aliases; + this.originalAliases = aliases; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 67f1a627ae77f..9e0014b0f512b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -10,7 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.TransportLocalClusterStateAction; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java deleted file mode 100644 index d48dc74295bcb..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.support; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ActionRunnable; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskManager; - -import java.util.concurrent.Executor; - -/** - * Analogue of {@link org.elasticsearch.action.support.master.TransportMasterNodeReadAction} except that it runs on the local node rather - * than delegating to the master. - */ -public abstract class TransportLocalClusterStateAction extends - TransportAction { - - protected final ClusterService clusterService; - protected final Executor executor; - - protected TransportLocalClusterStateAction( - String actionName, - ActionFilters actionFilters, - TaskManager taskManager, - ClusterService clusterService, - Executor executor - ) { - // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 - super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); - this.clusterService = clusterService; - this.executor = executor; - } - - protected abstract ClusterBlockException checkBlock(Request request, ClusterState state); - - @Override - protected final void doExecute(Task task, Request request, ActionListener listener) { - final var state = clusterService.state(); - final var clusterBlockException = checkBlock(request, state); - if (clusterBlockException != null) { - throw clusterBlockException; - } - - // Workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can - executor.execute(ActionRunnable.wrap(listener, l -> localClusterStateOperation(task, request, state, l))); - } - - protected abstract void localClusterStateOperation(Task task, Request request, ClusterState state, ActionListener listener) - throws Exception; -} diff --git a/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java new file mode 100644 index 0000000000000..dfbcb21c2a959 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/support/local/LocalClusterStateRequest.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support.local; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Objects; + +/** + * A base request for actions that are executed locally on the node that receives the request. + */ +public abstract class LocalClusterStateRequest extends ActionRequest { + + /** + * The timeout for waiting until the cluster is unblocked. + * We use the name masterTimeout to be consistent with the master node actions. + */ + private final TimeValue masterTimeout; + + protected LocalClusterStateRequest(TimeValue masterTimeout) { + this.masterTimeout = Objects.requireNonNull(masterTimeout); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + + public TimeValue masterTimeout() { + return masterTimeout; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java new file mode 100644 index 0000000000000..66f94050c9826 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/support/local/TransportLocalClusterStateAction.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support.local; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; + +import java.util.concurrent.Executor; + +import static org.elasticsearch.common.Strings.format; + +/** + * Analogue of {@link org.elasticsearch.action.support.master.TransportMasterNodeReadAction} except that it runs on the local node rather + * than delegating to the master. + */ +public abstract class TransportLocalClusterStateAction extends + TransportAction { + + private static final Logger logger = LogManager.getLogger(TransportLocalClusterStateAction.class); + + protected final ClusterService clusterService; + protected final Executor executor; + + protected TransportLocalClusterStateAction( + String actionName, + ActionFilters actionFilters, + TaskManager taskManager, + ClusterService clusterService, + Executor executor + ) { + // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 + super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.clusterService = clusterService; + this.executor = executor; + } + + protected abstract ClusterBlockException checkBlock(Request request, ClusterState state); + + protected abstract void localClusterStateOperation(Task task, Request request, ClusterState state, ActionListener listener) + throws Exception; + + @Override + protected final void doExecute(Task task, Request request, ActionListener listener) { + final var state = clusterService.state(); + final var clusterBlockException = checkBlock(request, state); + if (clusterBlockException != null) { + if (clusterBlockException.retryable() == false) { + listener.onFailure(clusterBlockException); + } else { + waitForClusterUnblock(task, request, listener, state, clusterBlockException); + } + } else { + innerDoExecute(task, request, listener, state); + } + } + + private void innerDoExecute(Task task, Request request, ActionListener listener, ClusterState state) { + if (task instanceof CancellableTask cancellableTask && cancellableTask.notifyIfCancelled(listener)) { + return; + } + // Workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can + executor.execute(ActionRunnable.wrap(listener, l -> localClusterStateOperation(task, request, state, l))); + } + + private void waitForClusterUnblock( + Task task, + Request request, + ActionListener listener, + ClusterState initialState, + ClusterBlockException exception + ) { + var observer = new ClusterStateObserver( + initialState, + clusterService, + request.masterTimeout(), + logger, + clusterService.threadPool().getThreadContext() + ); + observer.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + logger.trace("retrying with cluster state version [{}]", state.version()); + innerDoExecute(task, request, listener, state); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + logger.debug( + () -> format("timed out while waiting for cluster to unblock in [%s] (timeout [%s])", actionName, timeout), + exception + ); + listener.onFailure(new ElasticsearchTimeoutException("timed out while waiting for cluster to unblock", exception)); + } + }, clusterState -> isTaskCancelled(task) || checkBlock(request, clusterState) == null); + } + + private boolean isTaskCancelled(Task task) { + return task instanceof CancellableTask cancellableTask && cancellableTask.isCancelled(); + } +} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index dfe501f29ce2e..841d7979862dc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestBuilderListener; @@ -207,7 +208,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final boolean namesProvided = request.hasParam("name"); final String[] aliases = request.paramAsStringArrayOrEmptyIfAll("name"); - final GetAliasesRequest getAliasesRequest = new GetAliasesRequest(aliases); + final var masterNodeTimeout = RestUtils.getMasterNodeTimeout(request); + final GetAliasesRequest getAliasesRequest = new GetAliasesRequest(masterNodeTimeout, aliases); final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); getAliasesRequest.indices(indices); getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java index 6aa0b1c865682..cfb714c354f02 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.Table; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -47,9 +48,11 @@ public boolean allowSystemIndexAccessByDefault() { @Override protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { - final GetAliasesRequest getAliasesRequest = request.hasParam("alias") - ? new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) - : new GetAliasesRequest(); + final var masterNodeTimeout = RestUtils.getMasterNodeTimeout(request); + final GetAliasesRequest getAliasesRequest = new GetAliasesRequest( + masterNodeTimeout, + Strings.commaDelimitedListToStringArray(request.param("alias")) + ); getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions())); return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin() .indices() diff --git a/server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java new file mode 100644 index 0000000000000..ea71cefc6608c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/support/local/TransportLocalClusterStateActionTests.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support.local; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelHelper; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; +import static org.elasticsearch.test.ClusterServiceUtils.setState; + +public class TransportLocalClusterStateActionTests extends ESTestCase { + + private static ThreadPool threadPool; + + private ClusterService clusterService; + private TaskManager taskManager; + + @BeforeClass + public static void beforeClass() { + threadPool = new TestThreadPool(getTestClass().getName()); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + clusterService = createClusterService(threadPool); + taskManager = new TaskManager(clusterService.getSettings(), threadPool, Set.of()); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + clusterService.close(); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + public void testNoBlock() throws ExecutionException, InterruptedException { + var request = new Request(); + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), null, request, listener); + assertTrue(listener.isDone()); + listener.get(); + } + + public void testRetryAfterBlock() throws ExecutionException, InterruptedException { + var request = new Request(); + ClusterBlock block = new ClusterBlock(randomInt(), "", true, true, false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL); + var state = ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(block)).build(); + setState(clusterService, state); + + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), null, request, listener); + + assertFalse(listener.isDone()); + setState(clusterService, ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build()); + assertTrue(listener.isDone()); + listener.get(); + } + + public void testNonRetryableBlock() { + var request = new Request(); + ClusterBlock block = new ClusterBlock(randomInt(), "", false, true, false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL); + var state = ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(block)).build(); + setState(clusterService, state); + + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), null, request, listener); + + assertTrue(listener.isDone()); + var exception = assertThrows(ExecutionException.class, listener::get); + assertTrue(exception.getCause() instanceof ClusterBlockException); + } + + public void testTaskCancelledAfterBlock() { + var request = new Request(); + ClusterBlock block = new ClusterBlock(randomInt(), "", true, true, false, randomFrom(RestStatus.values()), ClusterBlockLevel.ALL); + var state = ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(block)).build(); + setState(clusterService, state); + + CancellableTask task = new CancellableTask(randomLong(), "test", Action.ACTION_NAME, "", TaskId.EMPTY_TASK_ID, Map.of()); + PlainActionFuture listener = new PlainActionFuture<>(); + ActionTestUtils.execute(new Action(taskManager, clusterService), task, request, listener); + + TaskCancelHelper.cancel(task, "test"); + assertFalse(listener.isDone()); + setState(clusterService, ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build()); + assertTrue(listener.isDone()); + var exception = assertThrows(ExecutionException.class, listener::get); + assertTrue(exception.getCause() instanceof TaskCancelledException); + } + + private static class Request extends LocalClusterStateRequest { + + protected Request() { + super(TEST_REQUEST_TIMEOUT); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + private static class Response extends ActionResponse { + + @Override + public void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + } + + private static class Action extends TransportLocalClusterStateAction { + static final String ACTION_NAME = "internal:testAction"; + + Action(TaskManager taskManager, ClusterService clusterService) { + super(ACTION_NAME, new ActionFilters(Set.of()), taskManager, clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE); + } + + @Override + protected void localClusterStateOperation(Task task, Request request, ClusterState state, ActionListener listener) + throws Exception { + listener.onResponse(new Response()); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + Set blocks = state.blocks().global(); + return blocks.isEmpty() ? null : new ClusterBlockException(blocks); + } + } +} From 9d9eed5bad0ecbaeaa6397b992eff46c5f653a7d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 4 Dec 2024 22:20:04 +1100 Subject: [PATCH 05/23] Assert for empty routing table when hiding state (#117870) Routing table should always be empty when hiding cluster state for not-recovered block. This PR adds an explicit assertion for it. --- .../java/org/elasticsearch/gateway/ClusterStateUpdaters.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java index 8d8e661b3a010..9c16bcd9d2cc2 100644 --- a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java +++ b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java @@ -143,6 +143,7 @@ public static ClusterState hideStateIfNotRecovered(ClusterState state) { .coordinationMetadata(state.metadata().coordinationMetadata()) .build(); + assert state.routingTable().indicesRouting().isEmpty() : "routing table is not empty: " + state.routingTable().indicesRouting(); return ClusterState.builder(state).metadata(metadata).blocks(blocks.build()).build(); } return state; From c7e985c3b61ef1aece751e674f5431fb06174935 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 4 Dec 2024 12:20:47 +0100 Subject: [PATCH 06/23] Support ST_ENVELOPE and related ST_XMIN, etc. (#116964) Support ST_ENVELOPE and related ST_XMIN, etc. Based on the PostGIS equivalents: https://postgis.net/docs/ST_Envelope.html https://postgis.net/docs/ST_XMin.html https://postgis.net/docs/ST_XMax.html https://postgis.net/docs/ST_YMin.html https://postgis.net/docs/ST_YMax.html --- docs/changelog/116964.yaml | 6 + .../description/st_envelope.asciidoc | 5 + .../functions/description/st_xmax.asciidoc | 5 + .../functions/description/st_xmin.asciidoc | 5 + .../functions/description/st_ymax.asciidoc | 5 + .../functions/description/st_ymin.asciidoc | 5 + .../functions/examples/st_envelope.asciidoc | 13 + .../esql/functions/examples/st_xmax.asciidoc | 13 + .../esql/functions/examples/st_xmin.asciidoc | 13 + .../esql/functions/examples/st_ymax.asciidoc | 13 + .../esql/functions/examples/st_ymin.asciidoc | 13 + .../kibana/definition/st_envelope.json | 61 +++ .../functions/kibana/definition/st_xmax.json | 61 +++ .../functions/kibana/definition/st_xmin.json | 61 +++ .../functions/kibana/definition/st_ymax.json | 61 +++ .../functions/kibana/definition/st_ymin.json | 61 +++ .../esql/functions/kibana/docs/st_envelope.md | 13 + .../esql/functions/kibana/docs/st_xmax.md | 15 + .../esql/functions/kibana/docs/st_xmin.md | 15 + .../esql/functions/kibana/docs/st_ymax.md | 15 + .../esql/functions/kibana/docs/st_ymin.md | 15 + .../functions/layout/st_envelope.asciidoc | 15 + .../esql/functions/layout/st_xmax.asciidoc | 15 + .../esql/functions/layout/st_xmin.asciidoc | 15 + .../esql/functions/layout/st_ymax.asciidoc | 15 + .../esql/functions/layout/st_ymin.asciidoc | 15 + .../functions/parameters/st_envelope.asciidoc | 6 + .../functions/parameters/st_xmax.asciidoc | 6 + .../functions/parameters/st_xmin.asciidoc | 6 + .../functions/parameters/st_ymax.asciidoc | 6 + .../functions/parameters/st_ymin.asciidoc | 6 + .../esql/functions/signature/st_envelope.svg | 1 + .../esql/functions/signature/st_xmax.svg | 1 + .../esql/functions/signature/st_xmin.svg | 1 + .../esql/functions/signature/st_ymax.svg | 1 + .../esql/functions/signature/st_ymin.svg | 1 + .../esql/functions/spatial-functions.asciidoc | 10 + .../esql/functions/types/st_envelope.asciidoc | 12 + .../esql/functions/types/st_xmax.asciidoc | 12 + .../esql/functions/types/st_xmin.asciidoc | 12 + .../esql/functions/types/st_ymax.asciidoc | 12 + .../esql/functions/types/st_ymin.asciidoc | 12 + .../utils/SpatialEnvelopeVisitor.java | 356 ++++++++++++++++++ .../utils/SpatialEnvelopeVisitorTests.java | 194 ++++++++++ .../main/resources/spatial_shapes.csv-spec | 41 ++ .../spatial/StEnvelopeFromWKBEvaluator.java | 126 +++++++ .../StEnvelopeFromWKBGeoEvaluator.java | 126 +++++++ .../spatial/StXMaxFromWKBEvaluator.java | 127 +++++++ .../spatial/StXMaxFromWKBGeoEvaluator.java | 127 +++++++ .../spatial/StXMinFromWKBEvaluator.java | 127 +++++++ .../spatial/StXMinFromWKBGeoEvaluator.java | 127 +++++++ .../spatial/StYMaxFromWKBEvaluator.java | 127 +++++++ .../spatial/StYMaxFromWKBGeoEvaluator.java | 127 +++++++ .../spatial/StYMinFromWKBEvaluator.java | 127 +++++++ .../spatial/StYMinFromWKBGeoEvaluator.java | 127 +++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../esql/expression/ExpressionWritables.java | 10 + .../function/EsqlFunctionRegistry.java | 10 + .../function/scalar/spatial/StEnvelope.java | 138 +++++++ .../function/scalar/spatial/StXMax.java | 123 ++++++ .../function/scalar/spatial/StXMin.java | 123 ++++++ .../function/scalar/spatial/StYMax.java | 123 ++++++ .../function/scalar/spatial/StYMin.java | 123 ++++++ .../scalar/spatial/StEnvelopeTests.java | 88 +++++ .../function/scalar/spatial/StXMaxTests.java | 75 ++++ .../function/scalar/spatial/StXMinTests.java | 75 ++++ .../function/scalar/spatial/StYMaxTests.java | 75 ++++ .../function/scalar/spatial/StYMinTests.java | 75 ++++ .../rest-api-spec/test/esql/60_usage.yml | 4 +- 69 files changed, 3558 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/116964.yaml create mode 100644 docs/reference/esql/functions/description/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/description/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/description/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/description/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/description/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/examples/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/st_envelope.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_xmax.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_xmin.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_ymax.json create mode 100644 docs/reference/esql/functions/kibana/definition/st_ymin.json create mode 100644 docs/reference/esql/functions/kibana/docs/st_envelope.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_xmax.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_xmin.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_ymax.md create mode 100644 docs/reference/esql/functions/kibana/docs/st_ymin.md create mode 100644 docs/reference/esql/functions/layout/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/layout/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/parameters/st_ymin.asciidoc create mode 100644 docs/reference/esql/functions/signature/st_envelope.svg create mode 100644 docs/reference/esql/functions/signature/st_xmax.svg create mode 100644 docs/reference/esql/functions/signature/st_xmin.svg create mode 100644 docs/reference/esql/functions/signature/st_ymax.svg create mode 100644 docs/reference/esql/functions/signature/st_ymin.svg create mode 100644 docs/reference/esql/functions/types/st_envelope.asciidoc create mode 100644 docs/reference/esql/functions/types/st_xmax.asciidoc create mode 100644 docs/reference/esql/functions/types/st_xmin.asciidoc create mode 100644 docs/reference/esql/functions/types/st_ymax.asciidoc create mode 100644 docs/reference/esql/functions/types/st_ymin.asciidoc create mode 100644 libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java create mode 100644 libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java diff --git a/docs/changelog/116964.yaml b/docs/changelog/116964.yaml new file mode 100644 index 0000000000000..2e3ecd06fa098 --- /dev/null +++ b/docs/changelog/116964.yaml @@ -0,0 +1,6 @@ +pr: 116964 +summary: "Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions" +area: ES|QL +type: feature +issues: + - 104875 diff --git a/docs/reference/esql/functions/description/st_envelope.asciidoc b/docs/reference/esql/functions/description/st_envelope.asciidoc new file mode 100644 index 0000000000000..6b7cf8d97538a --- /dev/null +++ b/docs/reference/esql/functions/description/st_envelope.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Determines the minimum bounding box of the supplied geometry. diff --git a/docs/reference/esql/functions/description/st_xmax.asciidoc b/docs/reference/esql/functions/description/st_xmax.asciidoc new file mode 100644 index 0000000000000..f33ec590bf2d4 --- /dev/null +++ b/docs/reference/esql/functions/description/st_xmax.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the maximum value of the `x` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value. diff --git a/docs/reference/esql/functions/description/st_xmin.asciidoc b/docs/reference/esql/functions/description/st_xmin.asciidoc new file mode 100644 index 0000000000000..b06cbfacde7bf --- /dev/null +++ b/docs/reference/esql/functions/description/st_xmin.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the minimum value of the `x` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value. diff --git a/docs/reference/esql/functions/description/st_ymax.asciidoc b/docs/reference/esql/functions/description/st_ymax.asciidoc new file mode 100644 index 0000000000000..f9475dd967562 --- /dev/null +++ b/docs/reference/esql/functions/description/st_ymax.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the maximum value of the `y` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value. diff --git a/docs/reference/esql/functions/description/st_ymin.asciidoc b/docs/reference/esql/functions/description/st_ymin.asciidoc new file mode 100644 index 0000000000000..7228c63a16030 --- /dev/null +++ b/docs/reference/esql/functions/description/st_ymin.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the minimum value of the `y` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value. diff --git a/docs/reference/esql/functions/examples/st_envelope.asciidoc b/docs/reference/esql/functions/examples/st_envelope.asciidoc new file mode 100644 index 0000000000000..df8c0ad5607fa --- /dev/null +++ b/docs/reference/esql/functions/examples/st_envelope.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_envelope] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_envelope-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_xmax.asciidoc b/docs/reference/esql/functions/examples/st_xmax.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_xmax.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_xmin.asciidoc b/docs/reference/esql/functions/examples/st_xmin.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_xmin.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_ymax.asciidoc b/docs/reference/esql/functions/examples/st_ymax.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_ymax.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_ymin.asciidoc b/docs/reference/esql/functions/examples/st_ymin.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_ymin.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/st_envelope.json b/docs/reference/esql/functions/kibana/definition/st_envelope.json new file mode 100644 index 0000000000000..6c00dda265ac7 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_envelope.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_envelope", + "description" : "Determines the minimum bounding box of the supplied geometry.", + "signatures" : [ + { + "params" : [ + { + "name" : "geometry", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "geo_shape" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| KEEP abbrev, airport, envelope" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_xmax.json b/docs/reference/esql/functions/kibana/definition/st_xmax.json new file mode 100644 index 0000000000000..7be22617c0992 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_xmax.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_xmax", + "description" : "Extracts the maximum value of the `x` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_xmin.json b/docs/reference/esql/functions/kibana/definition/st_xmin.json new file mode 100644 index 0000000000000..8052fdb861cea --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_xmin.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_xmin", + "description" : "Extracts the minimum value of the `x` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_ymax.json b/docs/reference/esql/functions/kibana/definition/st_ymax.json new file mode 100644 index 0000000000000..1a53f7388ea56 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_ymax.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_ymax", + "description" : "Extracts the maximum value of the `y` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_ymin.json b/docs/reference/esql/functions/kibana/definition/st_ymin.json new file mode 100644 index 0000000000000..e11722a8f9c07 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_ymin.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_ymin", + "description" : "Extracts the minimum value of the `y` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/st_envelope.md b/docs/reference/esql/functions/kibana/docs/st_envelope.md new file mode 100644 index 0000000000000..5f4c3e4809a82 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_envelope.md @@ -0,0 +1,13 @@ + + +### ST_ENVELOPE +Determines the minimum bounding box of the supplied geometry. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| KEEP abbrev, airport, envelope +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_xmax.md b/docs/reference/esql/functions/kibana/docs/st_xmax.md new file mode 100644 index 0000000000000..bbde89df76fd0 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_xmax.md @@ -0,0 +1,15 @@ + + +### ST_XMAX +Extracts the maximum value of the `x` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_xmin.md b/docs/reference/esql/functions/kibana/docs/st_xmin.md new file mode 100644 index 0000000000000..1a6cee7dcfd62 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_xmin.md @@ -0,0 +1,15 @@ + + +### ST_XMIN +Extracts the minimum value of the `x` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_ymax.md b/docs/reference/esql/functions/kibana/docs/st_ymax.md new file mode 100644 index 0000000000000..61c9b6c288ca5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_ymax.md @@ -0,0 +1,15 @@ + + +### ST_YMAX +Extracts the maximum value of the `y` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_ymin.md b/docs/reference/esql/functions/kibana/docs/st_ymin.md new file mode 100644 index 0000000000000..f5817f10f20a5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_ymin.md @@ -0,0 +1,15 @@ + + +### ST_YMIN +Extracts the minimum value of the `y` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/layout/st_envelope.asciidoc b/docs/reference/esql/functions/layout/st_envelope.asciidoc new file mode 100644 index 0000000000000..a20d4275e0c9f --- /dev/null +++ b/docs/reference/esql/functions/layout/st_envelope.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_envelope]] +=== `ST_ENVELOPE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_envelope.svg[Embedded,opts=inline] + +include::../parameters/st_envelope.asciidoc[] +include::../description/st_envelope.asciidoc[] +include::../types/st_envelope.asciidoc[] +include::../examples/st_envelope.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_xmax.asciidoc b/docs/reference/esql/functions/layout/st_xmax.asciidoc new file mode 100644 index 0000000000000..b0c5e7695521e --- /dev/null +++ b/docs/reference/esql/functions/layout/st_xmax.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_xmax]] +=== `ST_XMAX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_xmax.svg[Embedded,opts=inline] + +include::../parameters/st_xmax.asciidoc[] +include::../description/st_xmax.asciidoc[] +include::../types/st_xmax.asciidoc[] +include::../examples/st_xmax.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_xmin.asciidoc b/docs/reference/esql/functions/layout/st_xmin.asciidoc new file mode 100644 index 0000000000000..55fbad88c4cf0 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_xmin.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_xmin]] +=== `ST_XMIN` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_xmin.svg[Embedded,opts=inline] + +include::../parameters/st_xmin.asciidoc[] +include::../description/st_xmin.asciidoc[] +include::../types/st_xmin.asciidoc[] +include::../examples/st_xmin.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_ymax.asciidoc b/docs/reference/esql/functions/layout/st_ymax.asciidoc new file mode 100644 index 0000000000000..e1022de4ba664 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_ymax.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_ymax]] +=== `ST_YMAX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_ymax.svg[Embedded,opts=inline] + +include::../parameters/st_ymax.asciidoc[] +include::../description/st_ymax.asciidoc[] +include::../types/st_ymax.asciidoc[] +include::../examples/st_ymax.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_ymin.asciidoc b/docs/reference/esql/functions/layout/st_ymin.asciidoc new file mode 100644 index 0000000000000..65511e1925e27 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_ymin.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_ymin]] +=== `ST_YMIN` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_ymin.svg[Embedded,opts=inline] + +include::../parameters/st_ymin.asciidoc[] +include::../description/st_ymin.asciidoc[] +include::../types/st_ymin.asciidoc[] +include::../examples/st_ymin.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/st_envelope.asciidoc b/docs/reference/esql/functions/parameters/st_envelope.asciidoc new file mode 100644 index 0000000000000..a31c6a85de367 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_envelope.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`geometry`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_xmax.asciidoc b/docs/reference/esql/functions/parameters/st_xmax.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_xmax.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_xmin.asciidoc b/docs/reference/esql/functions/parameters/st_xmin.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_xmin.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_ymax.asciidoc b/docs/reference/esql/functions/parameters/st_ymax.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_ymax.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_ymin.asciidoc b/docs/reference/esql/functions/parameters/st_ymin.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_ymin.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/st_envelope.svg b/docs/reference/esql/functions/signature/st_envelope.svg new file mode 100644 index 0000000000000..885a60e6fd86f --- /dev/null +++ b/docs/reference/esql/functions/signature/st_envelope.svg @@ -0,0 +1 @@ +ST_ENVELOPE(geometry) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_xmax.svg b/docs/reference/esql/functions/signature/st_xmax.svg new file mode 100644 index 0000000000000..348d5a7f72763 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_xmax.svg @@ -0,0 +1 @@ +ST_XMAX(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_xmin.svg b/docs/reference/esql/functions/signature/st_xmin.svg new file mode 100644 index 0000000000000..13d479b0458be --- /dev/null +++ b/docs/reference/esql/functions/signature/st_xmin.svg @@ -0,0 +1 @@ +ST_XMIN(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_ymax.svg b/docs/reference/esql/functions/signature/st_ymax.svg new file mode 100644 index 0000000000000..e6ecb00185c84 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_ymax.svg @@ -0,0 +1 @@ +ST_YMAX(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_ymin.svg b/docs/reference/esql/functions/signature/st_ymin.svg new file mode 100644 index 0000000000000..ae722f1edc3d4 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_ymin.svg @@ -0,0 +1 @@ +ST_YMIN(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/spatial-functions.asciidoc b/docs/reference/esql/functions/spatial-functions.asciidoc index eee44d337b4c6..c6a8467b39996 100644 --- a/docs/reference/esql/functions/spatial-functions.asciidoc +++ b/docs/reference/esql/functions/spatial-functions.asciidoc @@ -15,6 +15,11 @@ * <> * <> * <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> // end::spatial_list[] include::layout/st_distance.asciidoc[] @@ -24,3 +29,8 @@ include::layout/st_contains.asciidoc[] include::layout/st_within.asciidoc[] include::layout/st_x.asciidoc[] include::layout/st_y.asciidoc[] +include::layout/st_envelope.asciidoc[] +include::layout/st_xmax.asciidoc[] +include::layout/st_xmin.asciidoc[] +include::layout/st_ymax.asciidoc[] +include::layout/st_ymin.asciidoc[] diff --git a/docs/reference/esql/functions/types/st_envelope.asciidoc b/docs/reference/esql/functions/types/st_envelope.asciidoc new file mode 100644 index 0000000000000..43355394c6015 --- /dev/null +++ b/docs/reference/esql/functions/types/st_envelope.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +geometry | result +cartesian_point | cartesian_shape +cartesian_shape | cartesian_shape +geo_point | geo_shape +geo_shape | geo_shape +|=== diff --git a/docs/reference/esql/functions/types/st_xmax.asciidoc b/docs/reference/esql/functions/types/st_xmax.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_xmax.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_xmin.asciidoc b/docs/reference/esql/functions/types/st_xmin.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_xmin.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_ymax.asciidoc b/docs/reference/esql/functions/types/st_ymax.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_ymax.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_ymin.asciidoc b/docs/reference/esql/functions/types/st_ymin.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_ymin.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java new file mode 100644 index 0000000000000..eee4a62c7d588 --- /dev/null +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -0,0 +1,356 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.geometry.utils; + +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; + +import java.util.Locale; +import java.util.Optional; + +/** + * This visitor is designed to determine the spatial envelope (or BBOX or MBR) of a potentially complex geometry. + * It has two modes: + *
    + *
  • + * Cartesian mode: The envelope is determined by the minimum and maximum x/y coordinates. + * Incoming BBOX geometries with minX > maxX are treated as invalid. + * Resulting BBOX geometries will always have minX <= maxX. + *
  • + *
  • + * Geographic mode: The envelope is determined by the minimum and maximum x/y coordinates, + * considering the possibility of wrapping the longitude around the dateline. + * A bounding box can be determined either by wrapping the longitude around the dateline or not, + * and the smaller bounding box is chosen. It is possible to disable the wrapping of the longitude. + *
+ * Usage of this is as simple as: + * + * Optional<Rectangle> bbox = SpatialEnvelopeVisitor.visit(geometry); + * if (bbox.isPresent()) { + * Rectangle envelope = bbox.get(); + * // Do stuff with the envelope + * } + * + * It is also possible to create the inner PointVisitor separately, as well as use the visitor for multiple geometries. + * + * PointVisitor pointVisitor = new CartesianPointVisitor(); + * SpatialEnvelopeVisitor visitor = new SpatialEnvelopeVisitor(pointVisitor); + * for (Geometry geometry : geometries) { + * geometry.visit(visitor); + * } + * if (visitor.isValid()) { + * Rectangle envelope = visitor.getResult(); + * // Do stuff with the envelope + * } + * + * Code that wishes to modify the behaviour of the visitor can implement the PointVisitor interface, + * or extend the existing implementations. + */ +public class SpatialEnvelopeVisitor implements GeometryVisitor { + + private final PointVisitor pointVisitor; + + public SpatialEnvelopeVisitor(PointVisitor pointVisitor) { + this.pointVisitor = pointVisitor; + } + + /** + * Determine the BBOX without considering the CRS or wrapping of the longitude. + * Note that incoming BBOX's that do cross the dateline (minx>maxx) will be treated as invalid. + */ + public static Optional visitCartesian(Geometry geometry) { + var visitor = new SpatialEnvelopeVisitor(new CartesianPointVisitor()); + if (geometry.visit(visitor)) { + return Optional.of(visitor.getResult()); + } + return Optional.empty(); + } + + /** + * Determine the BBOX assuming the CRS is geographic (eg WGS84) and optionally wrapping the longitude around the dateline. + */ + public static Optional visitGeo(Geometry geometry, boolean wrapLongitude) { + var visitor = new SpatialEnvelopeVisitor(new GeoPointVisitor(wrapLongitude)); + if (geometry.visit(visitor)) { + return Optional.of(visitor.getResult()); + } + return Optional.empty(); + } + + public Rectangle getResult() { + return pointVisitor.getResult(); + } + + /** + * Visitor for visiting points and rectangles. This is where the actual envelope calculation happens. + * There are two implementations, one for cartesian coordinates and one for geographic coordinates. + * The latter can optionally wrap the longitude around the dateline. + */ + public interface PointVisitor { + void visitPoint(double x, double y); + + void visitRectangle(double minX, double maxX, double maxY, double minY); + + boolean isValid(); + + Rectangle getResult(); + } + + /** + * The cartesian point visitor determines the envelope by the minimum and maximum x/y coordinates. + * It also disallows invalid rectangles where minX > maxX. + */ + public static class CartesianPointVisitor implements PointVisitor { + private double minX = Double.POSITIVE_INFINITY; + private double minY = Double.POSITIVE_INFINITY; + private double maxX = Double.NEGATIVE_INFINITY; + private double maxY = Double.NEGATIVE_INFINITY; + + public double getMinX() { + return minX; + } + + public double getMinY() { + return minY; + } + + public double getMaxX() { + return maxX; + } + + public double getMaxY() { + return maxY; + } + + @Override + public void visitPoint(double x, double y) { + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + + @Override + public void visitRectangle(double minX, double maxX, double maxY, double minY) { + if (minX > maxX) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid cartesian rectangle: minX (%s) > maxX (%s)", minX, maxX) + ); + } + this.minX = Math.min(this.minX, minX); + this.minY = Math.min(this.minY, minY); + this.maxX = Math.max(this.maxX, maxX); + this.maxY = Math.max(this.maxY, maxY); + } + + @Override + public boolean isValid() { + return minY != Double.POSITIVE_INFINITY; + } + + @Override + public Rectangle getResult() { + return new Rectangle(minX, maxX, maxY, minY); + } + } + + /** + * The geographic point visitor determines the envelope by the minimum and maximum x/y coordinates, + * while allowing for wrapping the longitude around the dateline. + * When longitude wrapping is enabled, the visitor will determine the smallest bounding box between the two choices: + *
    + *
  • Wrapping around the front of the earth, in which case the result will have minx < maxx
  • + *
  • Wrapping around the back of the earth, crossing the dateline, in which case the result will have minx > maxx
  • + *
+ */ + public static class GeoPointVisitor implements PointVisitor { + private double minY = Double.POSITIVE_INFINITY; + private double maxY = Double.NEGATIVE_INFINITY; + private double minNegX = Double.POSITIVE_INFINITY; + private double maxNegX = Double.NEGATIVE_INFINITY; + private double minPosX = Double.POSITIVE_INFINITY; + private double maxPosX = Double.NEGATIVE_INFINITY; + + public double getMinY() { + return minY; + } + + public double getMaxY() { + return maxY; + } + + public double getMinNegX() { + return minNegX; + } + + public double getMaxNegX() { + return maxNegX; + } + + public double getMinPosX() { + return minPosX; + } + + public double getMaxPosX() { + return maxPosX; + } + + private final boolean wrapLongitude; + + public GeoPointVisitor(boolean wrapLongitude) { + this.wrapLongitude = wrapLongitude; + } + + @Override + public void visitPoint(double x, double y) { + minY = Math.min(minY, y); + maxY = Math.max(maxY, y); + visitLongitude(x); + } + + @Override + public void visitRectangle(double minX, double maxX, double maxY, double minY) { + this.minY = Math.min(this.minY, minY); + this.maxY = Math.max(this.maxY, maxY); + visitLongitude(minX); + visitLongitude(maxX); + } + + private void visitLongitude(double x) { + if (x >= 0) { + minPosX = Math.min(minPosX, x); + maxPosX = Math.max(maxPosX, x); + } else { + minNegX = Math.min(minNegX, x); + maxNegX = Math.max(maxNegX, x); + } + } + + @Override + public boolean isValid() { + return minY != Double.POSITIVE_INFINITY; + } + + @Override + public Rectangle getResult() { + return getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); + } + + private static Rectangle getResult( + double minNegX, + double minPosX, + double maxNegX, + double maxPosX, + double maxY, + double minY, + boolean wrapLongitude + ) { + assert Double.isFinite(maxY); + if (Double.isInfinite(minPosX)) { + return new Rectangle(minNegX, maxNegX, maxY, minY); + } else if (Double.isInfinite(minNegX)) { + return new Rectangle(minPosX, maxPosX, maxY, minY); + } else if (wrapLongitude) { + double unwrappedWidth = maxPosX - minNegX; + double wrappedWidth = (180 - minPosX) - (-180 - maxNegX); + if (unwrappedWidth <= wrappedWidth) { + return new Rectangle(minNegX, maxPosX, maxY, minY); + } else { + return new Rectangle(minPosX, maxNegX, maxY, minY); + } + } else { + return new Rectangle(minNegX, maxPosX, maxY, minY); + } + } + } + + private boolean isValid() { + return pointVisitor.isValid(); + } + + @Override + public Boolean visit(Circle circle) throws RuntimeException { + // TODO: Support circle, if given CRS (needed for radius to x/y coordinate transformation) + throw new UnsupportedOperationException("Circle is not supported"); + } + + @Override + public Boolean visit(GeometryCollection collection) throws RuntimeException { + collection.forEach(geometry -> geometry.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(Line line) throws RuntimeException { + for (int i = 0; i < line.length(); i++) { + pointVisitor.visitPoint(line.getX(i), line.getY(i)); + } + return isValid(); + } + + @Override + public Boolean visit(LinearRing ring) throws RuntimeException { + for (int i = 0; i < ring.length(); i++) { + pointVisitor.visitPoint(ring.getX(i), ring.getY(i)); + } + return isValid(); + } + + @Override + public Boolean visit(MultiLine multiLine) throws RuntimeException { + multiLine.forEach(line -> line.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(MultiPoint multiPoint) throws RuntimeException { + for (int i = 0; i < multiPoint.size(); i++) { + visit(multiPoint.get(i)); + } + return isValid(); + } + + @Override + public Boolean visit(MultiPolygon multiPolygon) throws RuntimeException { + multiPolygon.forEach(polygon -> polygon.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(Point point) throws RuntimeException { + pointVisitor.visitPoint(point.getX(), point.getY()); + return isValid(); + } + + @Override + public Boolean visit(Polygon polygon) throws RuntimeException { + visit(polygon.getPolygon()); + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + visit(polygon.getHole(i)); + } + return isValid(); + } + + @Override + public Boolean visit(Rectangle rectangle) throws RuntimeException { + pointVisitor.visitRectangle(rectangle.getMinX(), rectangle.getMaxX(), rectangle.getMaxY(), rectangle.getMinY()); + return isValid(); + } +} diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java new file mode 100644 index 0000000000000..fc35df295e566 --- /dev/null +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.geometry.utils; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class SpatialEnvelopeVisitorTests extends ESTestCase { + + public void testVisitCartesianShape() { + for (int i = 0; i < 1000; i++) { + var geometry = ShapeTestUtils.randomGeometryWithoutCircle(0, false); + var bbox = SpatialEnvelopeVisitor.visitCartesian(geometry); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitGeoShapeNoWrap() { + for (int i = 0; i < 1000; i++) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + var bbox = SpatialEnvelopeVisitor.visitGeo(geometry, false); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitGeoShapeWrap() { + for (int i = 0; i < 1000; i++) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, true); + var bbox = SpatialEnvelopeVisitor.visitGeo(geometry, false); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitCartesianPoints() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.CartesianPointVisitor()); + double minX = Double.MAX_VALUE; + double minY = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + for (int i = 0; i < 1000; i++) { + var x = randomFloat(); + var y = randomFloat(); + var point = new Point(x, y); + visitor.visit(point); + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + var result = visitor.getResult(); + assertThat(i + ": " + point, result.getMinX(), equalTo(minX)); + assertThat(i + ": " + point, result.getMinY(), equalTo(minY)); + assertThat(i + ": " + point, result.getMaxX(), equalTo(maxX)); + assertThat(i + ": " + point, result.getMaxY(), equalTo(maxY)); + } + } + + public void testVisitGeoPointsNoWrapping() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(false)); + double minY = Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + double minX = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + for (int i = 0; i < 1000; i++) { + var point = GeometryTestUtils.randomPoint(); + visitor.visit(point); + minY = Math.min(minY, point.getY()); + maxY = Math.max(maxY, point.getY()); + minX = Math.min(minX, point.getX()); + maxX = Math.max(maxX, point.getX()); + var result = visitor.getResult(); + assertThat(i + ": " + point, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + point, result.getMinX(), equalTo(minX)); + assertThat(i + ": " + point, result.getMinY(), equalTo(minY)); + assertThat(i + ": " + point, result.getMaxX(), equalTo(maxX)); + assertThat(i + ": " + point, result.getMaxY(), equalTo(maxY)); + } + } + + public void testVisitGeoPointsWrapping() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(true)); + double minY = Double.POSITIVE_INFINITY; + double maxY = Double.NEGATIVE_INFINITY; + double minNegX = Double.POSITIVE_INFINITY; + double maxNegX = Double.NEGATIVE_INFINITY; + double minPosX = Double.POSITIVE_INFINITY; + double maxPosX = Double.NEGATIVE_INFINITY; + for (int i = 0; i < 1000; i++) { + var point = GeometryTestUtils.randomPoint(); + visitor.visit(point); + minY = Math.min(minY, point.getY()); + maxY = Math.max(maxY, point.getY()); + if (point.getX() >= 0) { + minPosX = Math.min(minPosX, point.getX()); + maxPosX = Math.max(maxPosX, point.getX()); + } else { + minNegX = Math.min(minNegX, point.getX()); + maxNegX = Math.max(maxNegX, point.getX()); + } + var result = visitor.getResult(); + if (Double.isInfinite(minPosX)) { + // Only negative x values were considered + assertRectangleResult(i + ": " + point, result, minNegX, maxNegX, maxY, minY, false); + } else if (Double.isInfinite(minNegX)) { + // Only positive x values were considered + assertRectangleResult(i + ": " + point, result, minPosX, maxPosX, maxY, minY, false); + } else { + // Both positive and negative x values exist, we need to decide which way to wrap the bbox + double unwrappedWidth = maxPosX - minNegX; + double wrappedWidth = (180 - minPosX) - (-180 - maxNegX); + if (unwrappedWidth <= wrappedWidth) { + // The smaller bbox is around the front of the planet, no dateline wrapping required + assertRectangleResult(i + ": " + point, result, minNegX, maxPosX, maxY, minY, false); + } else { + // The smaller bbox is around the back of the planet, dateline wrapping required (minx > maxx) + assertRectangleResult(i + ": " + point, result, minPosX, maxNegX, maxY, minY, true); + } + } + } + } + + public void testWillCrossDateline() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(true)); + visitor.visit(new Point(-90.0, 0.0)); + visitor.visit(new Point(90.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-89.0, 0.0)); + visitor.visit(new Point(89.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-100.0, 0.0)); + visitor.visit(new Point(100.0, 0.0)); + assertCrossesDateline(visitor, true); + visitor.visit(new Point(-70.0, 0.0)); + visitor.visit(new Point(70.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-120.0, 0.0)); + visitor.visit(new Point(120.0, 0.0)); + assertCrossesDateline(visitor, true); + } + + private void assertCrossesDateline(SpatialEnvelopeVisitor visitor, boolean crossesDateline) { + var result = visitor.getResult(); + if (crossesDateline) { + assertThat("Crosses dateline, minx>maxx", result.getMinX(), greaterThanOrEqualTo(result.getMaxX())); + } else { + assertThat("Does not cross dateline, minx 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static BytesRef evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StEnvelope.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StEnvelopeFromWKBEvaluator get(DriverContext context) { + return new StEnvelopeFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StEnvelopeFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..c61e825c0ee71 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeFromWKBGeoEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StEnvelope}. + * This class is generated. Do not edit it. + */ +public final class StEnvelopeFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StEnvelopeFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StEnvelopeFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantBytesRefBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendBytesRef(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static BytesRef evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StEnvelope.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + BytesRef value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static BytesRef evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StEnvelope.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StEnvelopeFromWKBGeoEvaluator get(DriverContext context) { + return new StEnvelopeFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StEnvelopeFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java new file mode 100644 index 0000000000000..0d51ef709c217 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMax}. + * This class is generated. Do not edit it. + */ +public final class StXMaxFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMaxFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMaxFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMaxFromWKBEvaluator get(DriverContext context) { + return new StXMaxFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMaxFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..3707bf421d550 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMax}. + * This class is generated. Do not edit it. + */ +public final class StXMaxFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMaxFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMaxFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMax.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMaxFromWKBGeoEvaluator get(DriverContext context) { + return new StXMaxFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMaxFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java new file mode 100644 index 0000000000000..699402ad68dee --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMin}. + * This class is generated. Do not edit it. + */ +public final class StXMinFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMinFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMinFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMinFromWKBEvaluator get(DriverContext context) { + return new StXMinFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMinFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..6a8c041595c1c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StXMin}. + * This class is generated. Do not edit it. + */ +public final class StXMinFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StXMinFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StXMinFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StXMin.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StXMinFromWKBGeoEvaluator get(DriverContext context) { + return new StXMinFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StXMinFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java new file mode 100644 index 0000000000000..e8b50099f38f6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMax}. + * This class is generated. Do not edit it. + */ +public final class StYMaxFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMaxFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMaxFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMaxFromWKBEvaluator get(DriverContext context) { + return new StYMaxFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMaxFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..00e75f862a86c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMax}. + * This class is generated. Do not edit it. + */ +public final class StYMaxFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMaxFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMaxFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMax.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMaxFromWKBGeoEvaluator get(DriverContext context) { + return new StYMaxFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMaxFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java new file mode 100644 index 0000000000000..cab66683261aa --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMin}. + * This class is generated. Do not edit it. + */ +public final class StYMinFromWKBEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMinFromWKBEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMinFromWKB"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinary(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinary(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMinFromWKBEvaluator get(DriverContext context) { + return new StYMinFromWKBEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMinFromWKBEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java new file mode 100644 index 0000000000000..8bae9d369fbb4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinFromWKBGeoEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StYMin}. + * This class is generated. Do not edit it. + */ +public final class StYMinFromWKBGeoEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public StYMinFromWKBGeoEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "StYMinFromWKBGeo"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantDoubleBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendDouble(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinaryGeo(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + double value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendDouble(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return StYMin.fromWellKnownBinaryGeo(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public StYMinFromWKBGeoEvaluator get(DriverContext context) { + return new StYMinFromWKBGeoEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "StYMinFromWKBGeoEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 4845c7061949b..19ba6a5151eaf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -214,6 +214,11 @@ public enum Cap { */ SPATIAL_CENTROID_NO_RECORDS, + /** + * Support ST_ENVELOPE function (and related ST_XMIN, etc.). + */ + ST_ENVELOPE, + /** * Fix to GROK and DISSECT that allows extracting attributes with the same name as the input * https://github.com/elastic/elasticsearch/issues/110184 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java index 7e2de0094c2ab..febeccdad9d78 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java @@ -57,8 +57,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StEnvelope; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StX; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StY; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMin; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; @@ -166,6 +171,11 @@ public static List unaryScalars() { entries.add(Sinh.ENTRY); entries.add(Space.ENTRY); entries.add(Sqrt.ENTRY); + entries.add(StEnvelope.ENTRY); + entries.add(StXMax.ENTRY); + entries.add(StXMin.ENTRY); + entries.add(StYMax.ENTRY); + entries.add(StYMin.ENTRY); entries.add(StX.ENTRY); entries.add(StY.ENTRY); entries.add(Tan.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 37b159922906c..c66a5293eb14a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -117,8 +117,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StEnvelope; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StX; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StXMin; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StY; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMax; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMin; import org.elasticsearch.xpack.esql.expression.function.scalar.string.BitLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; @@ -351,6 +356,11 @@ private static FunctionDefinition[][] functions() { def(SpatialIntersects.class, SpatialIntersects::new, "st_intersects"), def(SpatialWithin.class, SpatialWithin::new, "st_within"), def(StDistance.class, StDistance::new, "st_distance"), + def(StEnvelope.class, StEnvelope::new, "st_envelope"), + def(StXMax.class, StXMax::new, "st_xmax"), + def(StXMin.class, StXMin::new, "st_xmin"), + def(StYMax.class, StYMax::new, "st_ymax"), + def(StYMin.class, StYMin::new, "st_ymin"), def(StX.class, StX::new, "st_x"), def(StY.class, StY::new, "st_y") }, // conditional diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java new file mode 100644 index 0000000000000..934991f3a8088 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelope.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the minimum bounding rectangle of a geometry. + * The function `st_envelope` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at + * PostGIS:ST_ENVELOPE. + */ +public class StEnvelope extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "StEnvelope", + StEnvelope::new + ); + private DataType dataType; + + @FunctionInfo( + returnType = { "geo_shape", "cartesian_shape" }, + description = "Determines the minimum bounding box of the supplied geometry.", + examples = @Example(file = "spatial_shapes", tag = "st_envelope") + ) + public StEnvelope( + Source source, + @Param( + name = "geometry", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StEnvelope(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + var resolution = isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + if (resolution.resolved()) { + this.dataType = switch (field().dataType()) { + case GEO_POINT, GEO_SHAPE -> GEO_SHAPE; + case CARTESIAN_POINT, CARTESIAN_SHAPE -> CARTESIAN_SHAPE; + default -> NULL; + }; + } + return resolution; + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StEnvelopeFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StEnvelopeFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return dataType; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StEnvelope(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StEnvelope::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static BytesRef fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point) { + return wkb; + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return UNSPECIFIED.asWkb(envelope.get()); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static BytesRef fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point) { + return wkb; + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return UNSPECIFIED.asWkb(envelope.get()); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java new file mode 100644 index 0000000000000..d6d710b175113 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMax.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the maximum value of the x-coordinate from a geometry. + * The function `st_xmax` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_XMAX. + */ +public class StXMax extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StXMax", StXMax::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the maximum value of the `x` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StXMax( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StXMax(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StXMaxFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StXMaxFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StXMax(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StXMax::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMaxX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java new file mode 100644 index 0000000000000..a5fa11bc11b0f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMin.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the minimum value of the x-coordinate from a geometry. + * The function `st_xmin` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_XMIN. + */ +public class StXMin extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StXMin", StXMin::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the minimum value of the `x` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StXMin( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StXMin(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StXMinFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StXMinFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StXMin(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StXMin::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMinX(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java new file mode 100644 index 0000000000000..fbbea8e024a6b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMax.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the maximum value of the y-coordinate from a geometry. + * The function `st_ymax` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_YMAX. + */ +public class StYMax extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StYMax", StYMax::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the maximum value of the `y` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StYMax( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StYMax(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StYMaxFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StYMaxFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StYMax(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StYMax::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMaxY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java new file mode 100644 index 0000000000000..1707d3b4f2fb9 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMin.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; +import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; + +/** + * Determines the minimum value of the y-coordinate from a geometry. + * The function `st_ymin` is defined in the OGC Simple Feature Access standard. + * Alternatively it is well described in PostGIS documentation at PostGIS:ST_YMIN. + */ +public class StYMin extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StYMin", StYMin::new); + + @FunctionInfo( + returnType = "double", + description = "Extracts the minimum value of the `y` coordinates from the supplied geometry.\n" + + "If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value.", + examples = @Example(file = "spatial_shapes", tag = "st_x_y_min_max") + ) + public StYMin( + Source source, + @Param( + name = "point", + type = { "geo_point", "geo_shape", "cartesian_point", "cartesian_shape" }, + description = "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. " + + "If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private StYMin(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + return isSpatial(field(), sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + if (field().dataType() == GEO_POINT || field().dataType() == DataType.GEO_SHAPE) { + return new StYMinFromWKBGeoEvaluator.Factory(toEvaluator.apply(field()), source()); + } + return new StYMinFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new StYMin(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StYMin::new, field()); + } + + @ConvertEvaluator(extraName = "FromWKB", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinary(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } + + @ConvertEvaluator(extraName = "FromWKBGeo", warnExceptions = { IllegalArgumentException.class }) + static double fromWellKnownBinaryGeo(BytesRef wkb) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = SpatialEnvelopeVisitor.visitGeo(geometry, true); + if (envelope.isPresent()) { + return envelope.get().getMinY(); + } + throw new IllegalArgumentException("Cannot determine envelope of geometry"); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java new file mode 100644 index 0000000000000..ac87d45491447 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StEnvelopeTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_envelope") +public class StEnvelopeTests extends AbstractScalarFunctionTestCase { + public StEnvelopeTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StEnvelopeFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StEnvelopeFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, GEO_SHAPE, StEnvelopeTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint( + suppliers, + expectedCartesian, + CARTESIAN_SHAPE, + StEnvelopeTests::valueOfCartesian, + List.of() + ); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, GEO_SHAPE, StEnvelopeTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape( + suppliers, + expectedCartesian, + CARTESIAN_SHAPE, + StEnvelopeTests::valueOfCartesian, + List.of() + ); + return parameterSuppliersFromTypedDataWithDefaultChecks( + false, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static BytesRef valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static BytesRef valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static BytesRef valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point) { + return wkb; + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return UNSPECIFIED.asWkb(envelope.get()); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StEnvelope(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java new file mode 100644 index 0000000000000..dc6e61e44f599 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMaxTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_xmax") +public class StXMaxTests extends AbstractScalarFunctionTestCase { + public StXMaxTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StXMaxFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StXMaxFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StXMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StXMaxTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StXMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StXMaxTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxX(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StXMax(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java new file mode 100644 index 0000000000000..8c06d18b1e281 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXMinTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_xmin") +public class StXMinTests extends AbstractScalarFunctionTestCase { + public StXMinTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StXMinFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StXMinFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StXMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StXMinTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StXMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StXMinTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getX(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinX(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StXMin(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java new file mode 100644 index 0000000000000..7222d7517f7ff --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMaxTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_ymax") +public class StYMaxTests extends AbstractScalarFunctionTestCase { + public StYMaxTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StYMaxFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StYMaxFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StYMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StYMaxTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StYMaxTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StYMaxTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMaxY(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StYMax(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java new file mode 100644 index 0000000000000..843c7bb649114 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYMinTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; + +@FunctionName("st_ymin") +public class StYMinTests extends AbstractScalarFunctionTestCase { + public StYMinTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String expectedGeo = "StYMinFromWKBGeoEvaluator[field=Attribute[channel=0]]"; + String expectedCartesian = "StYMinFromWKBEvaluator[field=Attribute[channel=0]]"; + final List suppliers = new ArrayList<>(); + TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedGeo, DOUBLE, StYMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedCartesian, DOUBLE, StYMinTests::valueOfCartesian, List.of()); + TestCaseSupplier.forUnaryGeoShape(suppliers, expectedGeo, DOUBLE, StYMinTests::valueOfGeo, List.of()); + TestCaseSupplier.forUnaryCartesianShape(suppliers, expectedCartesian, DOUBLE, StYMinTests::valueOfCartesian, List.of()); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "geo_point, cartesian_point, geo_shape or cartesian_shape" + ); + } + + private static double valueOfGeo(BytesRef wkb) { + return valueOf(wkb, true); + } + + private static double valueOfCartesian(BytesRef wkb) { + return valueOf(wkb, false); + } + + private static double valueOf(BytesRef wkb, boolean geo) { + var geometry = UNSPECIFIED.wkbToGeometry(wkb); + if (geometry instanceof Point point) { + return point.getY(); + } + var envelope = geo ? SpatialEnvelopeVisitor.visitGeo(geometry, true) : SpatialEnvelopeVisitor.visitCartesian(geometry); + if (envelope.isPresent()) { + return envelope.get().getMinY(); + } + throw new IllegalArgumentException("Geometry is empty"); + } + + @Override + protected Expression build(Source source, List args) { + return new StYMin(source, args.get(0)); + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index c23b44c00bd14..26e3c8ed0ef47 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 122} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 127} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 119} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 123} # check the "sister" test above for a likely update to the same esql.functions length check From cd58030d46640e22c6e1a736256b5c2ad094cf89 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:28 +1100 Subject: [PATCH 07/23] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupIPFromIndexKeep ASYNC} #117974 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 17a7b26d1c091..1568bed8a4f30 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -244,6 +244,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupIPFromIndexKeep ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/117974 # Examples: # From 4389059bec3c3a2a0a267300b74fc7f7b7716394 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:43 +1100 Subject: [PATCH 08/23] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndex ASYNC} #117975 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1568bed8a4f30..e0d457b26cc73 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -247,6 +247,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupIPFromIndexKeep ASYNC} issue: https://github.com/elastic/elasticsearch/issues/117974 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndex ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/117975 # Examples: # From 91d0539c2a1f5926f083c7aee050b5e1b18b8566 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:05:56 +1100 Subject: [PATCH 09/23] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndexKeep ASYNC} #117976 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e0d457b26cc73..944ee0a4ae158 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -250,6 +250,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndex ASYNC} issue: https://github.com/elastic/elasticsearch/issues/117975 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndexKeep ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/117976 # Examples: # From 8134d0244ccac43a826d5282c380f68c5863dab3 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:13:21 +0100 Subject: [PATCH 10/23] Unmute #117893 (#117909) This test was muted erroneously. --- muted-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 944ee0a4ae158..c3fe9425def2d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -221,8 +221,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117815 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT issue: https://github.com/elastic/elasticsearch/issues/111319 -- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT - issue: https://github.com/elastic/elasticsearch/issues/117893 - class: org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilderTests method: testToQuery issue: https://github.com/elastic/elasticsearch/issues/117904 From 435a0cc2721a23f6c97e5102e86b5babf3b64b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 4 Dec 2024 13:23:27 +0100 Subject: [PATCH 11/23] CreateClassLoaderEntitlement + extensions to parse logic (#117754) --- .../policy/CreateClassLoaderEntitlement.java | 16 ++++ .../runtime/policy/PolicyParser.java | 81 ++++++++++++------- .../policy/PolicyParserFailureTests.java | 7 +- .../runtime/policy/PolicyParserTests.java | 39 +++++++++ 4 files changed, 109 insertions(+), 34 deletions(-) create mode 100644 libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java new file mode 100644 index 0000000000000..708e0b87711fe --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +public class CreateClassLoaderEntitlement implements Entitlement { + @ExternalEntitlement + public CreateClassLoaderEntitlement() {} + +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index ea6603af99925..0d1a7c14ece4b 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -19,22 +19,43 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; - -import static org.elasticsearch.entitlement.runtime.policy.PolicyParserException.newPolicyParserException; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A parser to parse policy files for entitlements. */ public class PolicyParser { - protected static final String entitlementPackageName = Entitlement.class.getPackage().getName(); + private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of(FileEntitlement.class, CreateClassLoaderEntitlement.class) + .collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); protected final XContentParser policyParser; protected final String policyName; + static String getEntitlementTypeName(Class entitlementClass) { + var entitlementClassName = entitlementClass.getSimpleName(); + + if (entitlementClassName.endsWith("Entitlement") == false) { + throw new IllegalArgumentException( + entitlementClassName + " is not a valid Entitlement class name. A valid class name must end with 'Entitlement'" + ); + } + + var strippedClassName = entitlementClassName.substring(0, entitlementClassName.indexOf("Entitlement")); + return Arrays.stream(strippedClassName.split("(?=\\p{Lu})")) + .filter(Predicate.not(String::isEmpty)) + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.joining("_")); + } + public PolicyParser(InputStream inputStream, String policyName) throws IOException { this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); this.policyName = policyName; @@ -67,18 +88,23 @@ protected Scope parseScope(String scopeName) throws IOException { } List entitlements = new ArrayList<>(); while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) { - if (policyParser.currentToken() != XContentParser.Token.START_OBJECT) { - throw newPolicyParserException(scopeName, "expected object "); - } - if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) { + if (policyParser.currentToken() == XContentParser.Token.VALUE_STRING) { + String entitlementType = policyParser.text(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + } else if (policyParser.currentToken() == XContentParser.Token.START_OBJECT) { + if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) { + throw newPolicyParserException(scopeName, "expected object "); + } + String entitlementType = policyParser.currentName(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + throw newPolicyParserException(scopeName, "expected closing object"); + } + } else { throw newPolicyParserException(scopeName, "expected object "); } - String entitlementType = policyParser.currentName(); - Entitlement entitlement = parseEntitlement(scopeName, entitlementType); - entitlements.add(entitlement); - if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { - throw newPolicyParserException(scopeName, "expected closing object"); - } } return new Scope(scopeName, entitlements); } catch (IOException ioe) { @@ -87,34 +113,29 @@ protected Scope parseScope(String scopeName) throws IOException { } protected Entitlement parseEntitlement(String scopeName, String entitlementType) throws IOException { - Class entitlementClass; - try { - entitlementClass = Class.forName( - entitlementPackageName - + "." - + Character.toUpperCase(entitlementType.charAt(0)) - + entitlementType.substring(1) - + "Entitlement" - ); - } catch (ClassNotFoundException cnfe) { - throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); - } - if (Entitlement.class.isAssignableFrom(entitlementClass) == false) { + Class entitlementClass = EXTERNAL_ENTITLEMENTS.get(entitlementType); + + if (entitlementClass == null) { throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); } + Constructor entitlementConstructor = entitlementClass.getConstructors()[0]; ExternalEntitlement entitlementMetadata = entitlementConstructor.getAnnotation(ExternalEntitlement.class); if (entitlementMetadata == null) { throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); } - if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { - throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); + Class[] parameterTypes = entitlementConstructor.getParameterTypes(); + String[] parametersNames = entitlementMetadata.parameterNames(); + + if (parameterTypes.length != 0 || parametersNames.length != 0) { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); + } } + Map parsedValues = policyParser.map(); - Class[] parameterTypes = entitlementConstructor.getParameterTypes(); - String[] parametersNames = entitlementMetadata.parameterNames(); Object[] parameterValues = new Object[parameterTypes.length]; for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) { String parameterName = parametersNames[parameterIndex]; diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java index de8280ea87fe5..7eb2b1fb476b3 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.nio.charset.StandardCharsets; public class PolicyParserFailureTests extends ESTestCase { @@ -26,7 +25,7 @@ public void testParserSyntaxFailures() { assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object ", ppe.getMessage()); } - public void testEntitlementDoesNotExist() throws IOException { + public void testEntitlementDoesNotExist() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - does_not_exist: {} @@ -38,7 +37,7 @@ public void testEntitlementDoesNotExist() throws IOException { ); } - public void testEntitlementMissingParameter() throws IOException { + public void testEntitlementMissingParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - file: {} @@ -61,7 +60,7 @@ public void testEntitlementMissingParameter() throws IOException { ); } - public void testEntitlementExtraneousParameter() throws IOException { + public void testEntitlementExtraneousParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - file: diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index 40016b2e3027e..a514cfe418895 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -11,11 +11,31 @@ import org.elasticsearch.test.ESTestCase; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + public class PolicyParserTests extends ESTestCase { + private static class TestWrongEntitlementName implements Entitlement {} + + public void testGetEntitlementTypeName() { + assertEquals("create_class_loader", PolicyParser.getEntitlementTypeName(CreateClassLoaderEntitlement.class)); + + var ex = expectThrows(IllegalArgumentException.class, () -> PolicyParser.getEntitlementTypeName(TestWrongEntitlementName.class)); + assertThat( + ex.getMessage(), + equalTo("TestWrongEntitlementName is not a valid Entitlement class name. A valid class name must end with 'Entitlement'") + ); + } + public void testPolicyBuilder() throws IOException { Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml") .parsePolicy(); @@ -25,4 +45,23 @@ public void testPolicyBuilder() throws IOException { ); assertEquals(parsedPolicy, builtPolicy); } + + public void testParseCreateClassloader() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - create_class_loader + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy(); + Policy builtPolicy = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement()))) + ); + assertThat( + parsedPolicy.scopes, + contains( + both(transformedMatch((Scope scope) -> scope.name, equalTo("entitlement-module-name"))).and( + transformedMatch(scope -> scope.entitlements, contains(instanceOf(CreateClassLoaderEntitlement.class))) + ) + ) + ); + } } From c09bdfb5a0215257e8c8cd1f7ef8ffa345503f56 Mon Sep 17 00:00:00 2001 From: Luke Whiting Date: Wed, 4 Dec 2024 12:27:42 +0000 Subject: [PATCH 12/23] Watcher history index has too many indexed fields - (#71479) (#117701) * Exclude result.input.chain from watcher history index mappings * Update docs/changelog/117701.yaml * Fixup text now fields are disabled higher up the chain * Revert priority change --- docs/changelog/117701.yaml | 6 ++++++ .../support/WatcherIndexTemplateRegistryField.java | 3 ++- .../src/main/resources/watch-history-no-ilm.json | 9 +++++++++ .../src/main/resources/watch-history.json | 9 +++++++++ .../test/integration/HistoryIntegrationTests.java | 6 +++--- 5 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/117701.yaml diff --git a/docs/changelog/117701.yaml b/docs/changelog/117701.yaml new file mode 100644 index 0000000000000..5a72bdeb143e6 --- /dev/null +++ b/docs/changelog/117701.yaml @@ -0,0 +1,6 @@ +pr: 117701 +summary: Watcher history index has too many indexed fields - +area: Watcher +type: bug +issues: + - 71479 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 20dcb84dffe3f..098549029e0ce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -22,8 +22,9 @@ public final class WatcherIndexTemplateRegistryField { // version 14: move watch history to data stream // version 15: remove watches and triggered watches, these are now system indices // version 16: change watch history ILM policy + // version 17: exclude input chain from indexing // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 16; + public static final int INDEX_TEMPLATE_VERSION = 17; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; public static final String[] TEMPLATE_NAMES = new String[] { HISTORY_TEMPLATE_NAME }; diff --git a/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json index 2eed69c7c58e6..da459cda13463 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/watch-history-no-ilm.json @@ -54,6 +54,15 @@ "enabled": false } } + }, + { + "disabled_result_input_chain_fields": { + "path_match": "result.input.chain", + "mapping": { + "type": "object", + "enabled": false + } + } } ], "dynamic": false, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json b/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json index 19e4dc022daa1..2abf6570d1f8e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/watch-history.json @@ -55,6 +55,15 @@ "enabled": false } } + }, + { + "disabled_result_input_chain_fields": { + "path_match": "result.input.chain", + "mapping": { + "type": "object", + "enabled": false + } + } } ], "dynamic": false, diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java index 0070554d99d27..1bcdd060994ce 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java @@ -130,7 +130,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio String chainedPath = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.chain.properties.chained.properties.search" + ".properties.request.properties.body.enabled"; - assertThat(source.getValue(chainedPath), is(false)); + assertThat(source.getValue(chainedPath), nullValue()); } else { String path = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.search.properties.request.properties.body.enabled"; @@ -168,11 +168,11 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception { XContentType.JSON ); - // lets make sure the body fields are disabled + // let's make sure the body fields are disabled or, in the case of chained, the whole object is not indexed if (useChained) { String path = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.chain.properties.chained.properties.payload.enabled"; - assertThat(source.getValue(path), is(false)); + assertThat(source.getValue(path), nullValue()); } else { String path = SINGLE_MAPPING_NAME + ".properties.result.properties.input.properties.payload.enabled"; assertThat(source.getValue(path), is(false)); From 59fd7bf502eebebf08f56cd73b376f5c5f171851 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:50:37 +1100 Subject: [PATCH 13/23] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {lookup-join.LookupMessageFromIndex SYNC} #117980 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index c3fe9425def2d..871afa5555f58 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,6 +251,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndexKeep ASYNC} issue: https://github.com/elastic/elasticsearch/issues/117976 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {lookup-join.LookupMessageFromIndex SYNC} + issue: https://github.com/elastic/elasticsearch/issues/117980 # Examples: # From 2244070da4f34d08f6bfc20150c86129251292e4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:50:57 +1100 Subject: [PATCH 14/23] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT #117981 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 871afa5555f58..651a1ee1ad1c9 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -254,6 +254,8 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {lookup-join.LookupMessageFromIndex SYNC} issue: https://github.com/elastic/elasticsearch/issues/117980 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + issue: https://github.com/elastic/elasticsearch/issues/117981 # Examples: # From 97601b1b0b1c2249fcd55d13994981afa6eeeab3 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Wed, 4 Dec 2024 08:10:56 -0500 Subject: [PATCH 15/23] Kderusso/sparse vector ci failure (#117930) * Fix CI failure in SparseVectorQueryBuilderTests * Unmute test --- muted-tests.yml | 3 --- .../xpack/core/ml/search/SparseVectorQueryBuilderTests.java | 6 ++++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 651a1ee1ad1c9..de11f0da32c44 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -221,9 +221,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117815 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT issue: https://github.com/elastic/elasticsearch/issues/111319 -- class: org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilderTests - method: testToQuery - issue: https://github.com/elastic/elasticsearch/issues/117904 - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test20NoAutoGenerationWhenAutoConfigurationDisabled issue: https://github.com/elastic/elasticsearch/issues/117891 diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java index 9872d95de024a..a5c1ba45d90b7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/search/SparseVectorQueryBuilderTests.java @@ -232,6 +232,12 @@ public void testToQuery() throws IOException { private void testDoToQuery(SparseVectorQueryBuilder queryBuilder, SearchExecutionContext context) throws IOException { Query query = queryBuilder.doToQuery(context); + + // test query builder can randomly have no vectors, which rewrites to a MatchNoneQuery - nothing more to do in this case. + if (query instanceof MatchNoDocsQuery) { + return; + } + assertTrue(query instanceof SparseVectorQueryWrapper); var sparseQuery = (SparseVectorQueryWrapper) query; if (queryBuilder.shouldPruneTokens()) { From ba1d5be199293d843414c5b1e4ea153ce651a06f Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:22:26 +0100 Subject: [PATCH 16/23] Updates minimum_number_of_allocations description (#117746) --- docs/reference/ml/ml-shared.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index d01047eac9815..4948db48664ed 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -18,7 +18,8 @@ end::adaptive-allocation-max-number[] tag::adaptive-allocation-min-number[] Specifies the minimum number of allocations to scale to. -If set, it must be greater than or equal to `1`. +If set, it must be greater than or equal to `0`. +If not defined, the deployment scales to `0`. end::adaptive-allocation-min-number[] tag::aggregations[] From 6855a4ecd3ea8989c995763dafcc0cdbcb25bc41 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:22:59 +0100 Subject: [PATCH 17/23] [DOCS] Adds adaptive allocations information to Inference APIs (#117546) * Adds adaptive allocations information to Inference APIs * Update docs/reference/inference/inference-apis.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> * Update docs/reference/inference/put-inference.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> * Update docs/reference/inference/inference-apis.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --------- Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- docs/reference/inference/inference-apis.asciidoc | 13 +++++++++++++ docs/reference/inference/put-inference.asciidoc | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 037d7abeb2a36..c7b779a994a05 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -35,6 +35,19 @@ Elastic –, then create an {infer} endpoint by the <>. Now use <> to perform <> on your data. +[discrete] +[[adaptive-allocations]] +=== Adaptive allocations + +Adaptive allocations allow inference services to dynamically adjust the number of model allocations based on the current load. + +When adaptive allocations are enabled: + +* The number of allocations scales up automatically when the load increases. +- Allocations scale down to a minimum of 0 when the load decreases, saving resources. + +For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation. + //[discrete] //[[default-enpoints]] //=== Default {infer} endpoints diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index e7e25ec98b49d..ed93c290b6ad4 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -67,4 +67,17 @@ Click the links to review the configuration details of the services: * <> (`text_embedding`) The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of -the services connect to external providers. \ No newline at end of file +the services connect to external providers. + +[discrete] +[[adaptive-allocations-put-inference]] +==== Adaptive allocations + +Adaptive allocations allow inference services to dynamically adjust the number of model allocations based on the current load. + +When adaptive allocations are enabled: + +- The number of allocations scales up automatically when the load increases. +- Allocations scale down to a minimum of 0 when the load decreases, saving resources. + +For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation. \ No newline at end of file From ec27e20c80f402e09822f5e1b4ebc7b02fc6196d Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 4 Dec 2024 07:37:57 -0600 Subject: [PATCH 18/23] Removing unnecessary state from DataStreamReindexTask (#117942) --- ...indexDataStreamPersistentTaskExecutor.java | 9 ++--- .../migrate/task/ReindexDataStreamTask.java | 35 ++++++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java index e2a41ea186643..0f3f8b17f27ad 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java @@ -51,7 +51,6 @@ protected ReindexDataStreamTask createTask( params.startTime(), params.totalIndices(), params.totalIndicesToBeUpgraded(), - threadPool, id, type, action, @@ -74,9 +73,11 @@ protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTask List indicesToBeReindexed = indices.stream() .filter(index -> clusterService.state().getMetadata().index(index).getCreationVersion().isLegacyIndexVersion()) .toList(); - reindexDataStreamTask.setPendingIndices(indicesToBeReindexed.stream().map(Index::getName).toList()); + reindexDataStreamTask.setPendingIndicesCount(indicesToBeReindexed.size()); for (Index index : indicesToBeReindexed) { + reindexDataStreamTask.incrementInProgressIndicesCount(); // TODO This is just a placeholder. This is where the real data stream reindex logic will go + reindexDataStreamTask.reindexSucceeded(); } completeSuccessfulPersistentTask(reindexDataStreamTask); @@ -87,12 +88,12 @@ protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTask } private void completeSuccessfulPersistentTask(ReindexDataStreamTask persistentTask) { - persistentTask.reindexSucceeded(); + persistentTask.allReindexesCompleted(); threadPool.schedule(persistentTask::markAsCompleted, getTimeToLive(persistentTask), threadPool.generic()); } private void completeFailedPersistentTask(ReindexDataStreamTask persistentTask, Exception e) { - persistentTask.reindexFailed(e); + persistentTask.taskFailed(e); threadPool.schedule(() -> persistentTask.markAsFailed(e), getTimeToLive(persistentTask), threadPool.generic()); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java index 722b30d9970db..72ddb87e9dea5 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java @@ -10,29 +10,27 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.persistent.AllocatedPersistentTask; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public class ReindexDataStreamTask extends AllocatedPersistentTask { public static final String TASK_NAME = "reindex-data-stream"; private final long persistentTaskStartTime; private final int totalIndices; private final int totalIndicesToBeUpgraded; - private final ThreadPool threadPool; private boolean complete = false; private Exception exception; - private List inProgress = new ArrayList<>(); - private List pending = List.of(); + private AtomicInteger inProgress = new AtomicInteger(0); + private AtomicInteger pending = new AtomicInteger(); private List> errors = new ArrayList<>(); public ReindexDataStreamTask( long persistentTaskStartTime, int totalIndices, int totalIndicesToBeUpgraded, - ThreadPool threadPool, long id, String type, String action, @@ -44,7 +42,6 @@ public ReindexDataStreamTask( this.persistentTaskStartTime = persistentTaskStartTime; this.totalIndices = totalIndices; this.totalIndicesToBeUpgraded = totalIndicesToBeUpgraded; - this.threadPool = threadPool; } @Override @@ -55,30 +52,36 @@ public ReindexDataStreamStatus getStatus() { totalIndicesToBeUpgraded, complete, exception, - inProgress.size(), - pending.size(), + inProgress.get(), + pending.get(), errors ); } - public void reindexSucceeded() { + public void allReindexesCompleted() { this.complete = true; } - public void reindexFailed(Exception e) { + public void taskFailed(Exception e) { this.complete = true; this.exception = e; } - public void setInProgressIndices(List inProgressIndices) { - this.inProgress = inProgressIndices; + public void reindexSucceeded() { + inProgress.decrementAndGet(); + } + + public void reindexFailed(String index, Exception error) { + this.errors.add(Tuple.tuple(index, error)); + inProgress.decrementAndGet(); } - public void setPendingIndices(List pendingIndices) { - this.pending = pendingIndices; + public void incrementInProgressIndicesCount() { + inProgress.incrementAndGet(); + pending.decrementAndGet(); } - public void addErrorIndex(String index, Exception error) { - this.errors.add(Tuple.tuple(index, error)); + public void setPendingIndicesCount(int size) { + pending.set(size); } } From bbf986dd2c46a1ea4fec2eabe12b5706ab4f3682 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Dec 2024 10:20:49 -0500 Subject: [PATCH 19/23] ESQL: Limit size of query (#117898) Queries bigger than a mb tend to take a lot of memory. In the worse case it's an astounding amount of memory. --- docs/changelog/117898.yaml | 5 +++++ .../xpack/esql/heap_attack/HeapAttackIT.java | 21 +++++++++++++++++++ .../xpack/esql/parser/EsqlParser.java | 17 ++++++++++++++- .../xpack/esql/analysis/ParsingTests.java | 8 +++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/117898.yaml diff --git a/docs/changelog/117898.yaml b/docs/changelog/117898.yaml new file mode 100644 index 0000000000000..c60061abc49ff --- /dev/null +++ b/docs/changelog/117898.yaml @@ -0,0 +1,5 @@ +pr: 117898 +summary: Limit size of query +area: ES|QL +type: bug +issues: [] diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 8b9176a346e30..ace3db377664c 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -194,6 +194,13 @@ private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { ); } + private void assertParseFailure(ThrowingRunnable r) throws IOException { + ResponseException e = expectThrows(ResponseException.class, r); + Map map = responseAsMap(e.getResponse()); + logger.info("expected parse failure {}", map); + assertMap(map, matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "parsing_exception"))); + } + private Response sortByManyLongs(int count) throws IOException { logger.info("sorting by {} longs", count); return query(makeSortByManyLongs(count).toString(), null); @@ -318,6 +325,13 @@ public void testManyConcatFromRow() throws IOException { assertManyStrings(resp, strings); } + /** + * Fails to parse a huge huge query. + */ + public void testHugeHugeManyConcatFromRow() throws IOException { + assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 50000)); + } + /** * Tests that generate many moderately long strings. */ @@ -378,6 +392,13 @@ public void testManyRepeatFromRow() throws IOException { assertManyStrings(resp, strings); } + /** + * Fails to parse a huge huge query. + */ + public void testHugeHugeManyRepeatFromRow() throws IOException { + assertParseFailure(() -> manyRepeat("ROW a = 99", 100000)); + } + /** * Tests that generate many moderately long strings. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 620a25e0170ea..2e55b4df1e223 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -33,6 +33,15 @@ public class EsqlParser { private static final Logger log = LogManager.getLogger(EsqlParser.class); + /** + * Maximum number of characters in an ESQL query. Antlr may parse the entire + * query into tokens to make the choices, buffering the world. There's a lot we + * can do in the grammar to prevent that, but let's be paranoid and assume we'll + * fail at preventing antlr from slurping in the world. Instead, let's make sure + * that the world just isn't that big. + */ + public static final int MAX_LENGTH = 1_000_000; + private EsqlConfig config = new EsqlConfig(); public EsqlConfig config() { @@ -60,8 +69,14 @@ private T invokeParser( Function parseFunction, BiFunction result ) { + if (query.length() > MAX_LENGTH) { + throw new org.elasticsearch.xpack.esql.core.ParsingException( + "ESQL statement is too large [{} characters > {}]", + query.length(), + MAX_LENGTH + ); + } try { - // new CaseChangingCharStream() EsqlBaseLexer lexer = new EsqlBaseLexer(CharStreams.fromString(query)); lexer.removeErrorListeners(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 3cafd42b731f6..68529e99c6b1b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -103,6 +103,14 @@ public void testInlineCast() throws IOException { logger.info("Wrote to file: {}", file); } + public void testTooBigQuery() { + StringBuilder query = new StringBuilder("FROM foo | EVAL a = a"); + while (query.length() < EsqlParser.MAX_LENGTH) { + query.append(", a = CONCAT(a, a)"); + } + assertEquals("-1:0: ESQL statement is too large [1000011 characters > 1000000]", error(query.toString())); + } + private String functionName(EsqlFunctionRegistry registry, Expression functionCall) { for (FunctionDefinition def : registry.listFunctions()) { if (functionCall.getClass().equals(def.clazz())) { From cb35dc96f689e1e7ed4b7ce21a9923ce6d796405 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 4 Dec 2024 07:30:26 -0800 Subject: [PATCH 20/23] Make Murmur3Hasher#update public (#117961) Closes #117883 --- .../java/org/elasticsearch/common/hash/Murmur3Hasher.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java b/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java index 817587771d795..aec28484138fb 100644 --- a/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java +++ b/server/src/main/java/org/elasticsearch/common/hash/Murmur3Hasher.java @@ -40,7 +40,12 @@ public void update(byte[] inputBytes) { update(inputBytes, 0, inputBytes.length); } - private void update(byte[] inputBytes, int offset, int length) { + /** + * Similar to {@link #update(byte[])}, but processes a specific portion of the input bytes + * starting from the given {@code offset} for the specified {@code length}. + * @see #update(byte[]) + */ + public void update(byte[] inputBytes, int offset, int length) { if (remainderLength + length >= remainder.length) { if (remainderLength > 0) { // fill rest of remainder from inputBytes and hash remainder From ef11b44b4319e0f5cfef9186519ed1f7dc2fede2 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 4 Dec 2024 07:32:00 -0800 Subject: [PATCH 21/23] Acquire stats searcher for data stream stats (#117953) Here, we only need to extract the minimum and maximum values of the timestamp field; therefore, using a stats searcher should suffice. This is important for frozen indices. --- docs/changelog/117953.yaml | 5 +++++ .../datastreams/action/DataStreamsStatsTransportAction.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/117953.yaml diff --git a/docs/changelog/117953.yaml b/docs/changelog/117953.yaml new file mode 100644 index 0000000000000..62f0218b1cdc7 --- /dev/null +++ b/docs/changelog/117953.yaml @@ -0,0 +1,5 @@ +pr: 117953 +summary: Acquire stats searcher for data stream stats +area: Data streams +type: bug +issues: [] diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index 1b0b0aa6abebe..1d3b1b676282a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.indices.IndicesService; @@ -130,7 +131,7 @@ protected void shardOperation( DataStream dataStream = indexAbstraction.getParentDataStream(); assert dataStream != null; long maxTimestamp = 0L; - try (Engine.Searcher searcher = indexShard.acquireSearcher("data_stream_stats")) { + try (Engine.Searcher searcher = indexShard.acquireSearcher(ReadOnlyEngine.FIELD_RANGE_SEARCH_SOURCE)) { IndexReader indexReader = searcher.getIndexReader(); byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, DataStream.TIMESTAMP_FIELD_NAME); if (maxPackedValue != null) { From 6b329907dada4ad29a4093bf00ecb71319b95cfd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 02:36:05 +1100 Subject: [PATCH 22/23] Mute org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests test30NoAutogenerationWhenDaemonized #117956 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index de11f0da32c44..beb8cb9591b4f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -253,6 +253,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/117980 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT issue: https://github.com/elastic/elasticsearch/issues/117981 +- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests + method: test30NoAutogenerationWhenDaemonized + issue: https://github.com/elastic/elasticsearch/issues/117956 # Examples: # From 94f65797ccb0378779550a2d4fca2b1528d65197 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 5 Dec 2024 02:36:09 +1100 Subject: [PATCH 23/23] Mute org.elasticsearch.packaging.test.CertGenCliTests test40RunWithCert #117955 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index beb8cb9591b4f..317b960f36c56 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -256,6 +256,9 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test30NoAutogenerationWhenDaemonized issue: https://github.com/elastic/elasticsearch/issues/117956 +- class: org.elasticsearch.packaging.test.CertGenCliTests + method: test40RunWithCert + issue: https://github.com/elastic/elasticsearch/issues/117955 # Examples: #