From 5bc03b90332715e9ba483af778cc08804b05cd8b Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 8 Aug 2024 11:52:42 +0200 Subject: [PATCH 01/37] Extend logging for dropped warning headers. (#111624) This PR adds selected headers (xOpaqueId, product origin) when logging to help better understand the source of requests triggering such warnings. Closes #90527 --- docs/changelog/111624.yaml | 6 +++ .../elasticsearch/common/ReferenceDocs.java | 1 + .../common/util/concurrent/ThreadContext.java | 49 ++++++++++-------- .../common/reference-docs-links.json | 3 +- .../util/concurrent/ThreadContextTests.java | 51 +++++++++++++++++++ 5 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 docs/changelog/111624.yaml diff --git a/docs/changelog/111624.yaml b/docs/changelog/111624.yaml new file mode 100644 index 0000000000000..7b04b244ef7a7 --- /dev/null +++ b/docs/changelog/111624.yaml @@ -0,0 +1,6 @@ +pr: 111624 +summary: Extend logging for dropped warning headers +area: Infra/Core +type: enhancement +issues: + - 90527 diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index c11d369e3cc76..f710ae7c3b84a 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -80,6 +80,7 @@ public enum ReferenceDocs { LUCENE_MAX_DOCS_LIMIT, MAX_SHARDS_PER_NODE, FLOOD_STAGE_WATERMARK, + X_OPAQUE_ID, // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner ; diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java index 7caf570806c0e..0ff0bb2657a5c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -798,6 +799,30 @@ private ThreadContextStruct putResponseHeaders(Map> headers) return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext); } + private void logWarningHeaderThresholdExceeded(long threshold, Setting thresholdSetting) { + // If available, log some selected headers to help identifying the source of the request. + // Note: Only Task.HEADERS_TO_COPY are guaranteed to be preserved at this point. + Map selectedHeaders = new HashMap<>(requestHeaders); + selectedHeaders.keySet().retainAll(List.of(Task.X_OPAQUE_ID_HTTP_HEADER, Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER)); + + boolean sizeExceeded = HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.equals(thresholdSetting); + + StringBuilder message = new StringBuilder().append("Dropping a warning header"); + if (selectedHeaders.isEmpty() == false) { + message.append(" for request [").append(selectedHeaders).append("]"); + } + message.append(", as their total").append(sizeExceeded ? " size" : " count"); + message.append(" reached the maximum allowed of [").append(threshold).append("]").append(sizeExceeded ? " bytes" : ""); + message.append(" set in [").append(thresholdSetting.getKey()).append("]!"); + + if (selectedHeaders.containsKey(Task.X_OPAQUE_ID_HTTP_HEADER) == false) { + message.append(" Add X-Opaque-Id headers to identify the source of this warning"); + message.append(", see ").append(ReferenceDocs.X_OPAQUE_ID).append(" for more information."); + } + + logger.warn(message); + } + private ThreadContextStruct putResponse( final String key, final String value, @@ -810,24 +835,12 @@ private ThreadContextStruct putResponse( // check if we can add another warning header - if max size within limits if (key.equals("Warning") && (maxWarningHeaderSize != -1)) { // if size is NOT unbounded, check its limits if (warningHeadersSize > maxWarningHeaderSize) { // if max size has already been reached before - logger.warn( - "Dropping a warning header, as their total size reached the maximum allowed of [" - + maxWarningHeaderSize - + "] bytes set in [" - + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() - + "]!" - ); + logWarningHeaderThresholdExceeded(maxWarningHeaderSize, HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE); return this; } newWarningHeaderSize += "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; if (newWarningHeaderSize > maxWarningHeaderSize) { - logger.warn( - "Dropping a warning header, as their total size reached the maximum allowed of [" - + maxWarningHeaderSize - + "] bytes set in [" - + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() - + "]!" - ); + logWarningHeaderThresholdExceeded(maxWarningHeaderSize, HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE); return new ThreadContextStruct( requestHeaders, responseHeaders, @@ -857,13 +870,7 @@ private ThreadContextStruct putResponse( if ((key.equals("Warning")) && (maxWarningHeaderCount != -1)) { // if count is NOT unbounded, check its limits final int warningHeaderCount = newResponseHeaders.containsKey("Warning") ? newResponseHeaders.get("Warning").size() : 0; if (warningHeaderCount > maxWarningHeaderCount) { - logger.warn( - "Dropping a warning header, as their total count reached the maximum allowed of [" - + maxWarningHeaderCount - + "] set in [" - + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() - + "]!" - ); + logWarningHeaderThresholdExceeded(maxWarningHeaderCount, HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT); return this; } } diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index e325aea9f1089..8288ca792b0f1 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -40,5 +40,6 @@ "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services", "LUCENE_MAX_DOCS_LIMIT": "size-your-shards.html#troubleshooting-max-docs-limit", "MAX_SHARDS_PER_NODE": "size-your-shards.html#troubleshooting-max-shards-open", - "FLOOD_STAGE_WATERMARK": "fix-watermark-errors.html" + "FLOOD_STAGE_WATERMARK": "fix-watermark-errors.html", + "X_OPAQUE_ID": "api-conventions.html#x-opaque-id" } diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java index 11a871e9fbbce..88e3125655df0 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java @@ -7,12 +7,16 @@ */ package org.elasticsearch.common.util.concurrent; +import org.apache.logging.log4j.Level; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import java.io.IOException; import java.util.Arrays; @@ -28,6 +32,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLengthBetween; import static org.elasticsearch.tasks.Task.HEADERS_TO_COPY; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; @@ -633,6 +638,52 @@ public void testResponseHeaders() { } } + public void testDropWarningsExceedingMaxSettings() { + Settings settings = Settings.builder() + .put(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey(), 1) + .put(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey(), "50b") + .build(); + + try (var mockLog = MockLog.capture(ThreadContext.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "too many warnings", + ThreadContext.class.getCanonicalName(), + Level.WARN, + "Dropping a warning header,* count reached the maximum allowed of [1] set in [http.max_warning_header_count]!" + + ("* X-Opaque-Id header*, see " + ReferenceDocs.X_OPAQUE_ID + "*") + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "warnings too long", + ThreadContext.class.getCanonicalName(), + Level.WARN, + "Dropping a warning header for request [{X-Opaque-Id=abc, X-elastic-product-origin=product}], " + + "* size reached the maximum allowed of [50] bytes set in [http.max_warning_header_size]!" + ) + ); + + final ThreadContext threadContext = new ThreadContext(settings); + + threadContext.addResponseHeader("Warning", "warning"); + threadContext.addResponseHeader("Warning", "dropped, too many"); + + threadContext.putHeader(Task.X_OPAQUE_ID_HTTP_HEADER, "abc"); + threadContext.putHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, "product"); + threadContext.putHeader("other", "filtered out"); + + threadContext.addResponseHeader("Warning", "dropped, size exceeded " + randomAlphaOfLength(50)); + + final Map> responseHeaders = threadContext.getResponseHeaders(); + final List warnings = responseHeaders.get("Warning"); + + assertThat(warnings, contains("warning")); + mockLog.assertAllExpectationsMatched(); + } + + } + public void testCopyHeaders() { Settings build = Settings.builder().put("request.headers.default", "1").build(); ThreadContext threadContext = new ThreadContext(build); From bf0bcf4cc2329a3835ffb73aafb8bb7f59da974c Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 8 Aug 2024 13:25:52 +0300 Subject: [PATCH 02/37] Unmute tests that were fixed in the meantime (#111705) These were fixed by @not-napoleon, but CI caught them in the meantime and muted them nonetheless. --- muted-tests.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e05da61c1d44a..018685a194db7 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -122,18 +122,6 @@ tests: - class: org.elasticsearch.xpack.inference.integration.ModelRegistryIT method: testGetModel issue: https://github.com/elastic/elasticsearch/issues/111570 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testMatchCommandWithWhereClause {default} - issue: https://github.com/elastic/elasticsearch/issues/111658 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testMatchCommand {default} - issue: https://github.com/elastic/elasticsearch/issues/111659 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testMatchCommandWithMultipleMatches {default} - issue: https://github.com/elastic/elasticsearch/issues/111660 -- class: org.elasticsearch.xpack.esql.analysis.VerifierTests - method: testMatchCommand - issue: https://github.com/elastic/elasticsearch/issues/111661 # Examples: # From 76237b1e4594e6a58686ac37d4e06233e68e6703 Mon Sep 17 00:00:00 2001 From: Henning Andersen <33268011+henningandersen@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:47:13 +0200 Subject: [PATCH 03/37] Unsafe future fix check (#111703) UnsafePlainActionFuture would allow any thread to do unsafe completes when using the one-arg constructor, fixed. --- .../action/support/PlainActionFuture.java | 10 ++-- .../support/UnsafePlainActionFuture.java | 20 +++++-- .../support/UnsafePlainActionFutureTests.java | 57 +++++++++++++++++++ .../action/support/TestPlainActionFuture.java | 2 +- 4 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java diff --git a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java index ee4433369f689..4022ef3873771 100644 --- a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java @@ -381,12 +381,12 @@ private boolean assertCompleteAllowed() { } // only used in assertions - boolean allowedExecutors(Thread thread1, Thread thread2) { + boolean allowedExecutors(Thread blockedThread, Thread completingThread) { // this should only be used to validate thread interactions, like not waiting for a future completed on the same // executor, hence calling it with the same thread indicates a bug in the assertion using this. - assert thread1 != thread2 : "only call this for different threads"; - String thread1Name = EsExecutors.executorName(thread1); - String thread2Name = EsExecutors.executorName(thread2); - return thread1Name == null || thread2Name == null || thread1Name.equals(thread2Name) == false; + assert blockedThread != completingThread : "only call this for different threads"; + String blockedThreadName = EsExecutors.executorName(blockedThread); + String completingThreadName = EsExecutors.executorName(completingThread); + return blockedThreadName == null || completingThreadName == null || blockedThreadName.equals(completingThreadName) == false; } } diff --git a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java index 8aa6bc4de109a..dfbd4f2b1801a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java @@ -25,21 +25,29 @@ public class UnsafePlainActionFuture extends PlainActionFuture { private final String unsafeExecutor; private final String unsafeExecutor2; + /** + * Allow the single executor passed to be used unsafely. This allows waiting for the future and completing the future on threads in + * the same executor, but only for the specific executor. + */ public UnsafePlainActionFuture(String unsafeExecutor) { - this(unsafeExecutor, null); + this(unsafeExecutor, "__none__"); } + /** + * Allow both executors passed to be used unsafely. This allows waiting for the future and completing the future on threads in + * the same executor, but only for the two specific executors. + */ public UnsafePlainActionFuture(String unsafeExecutor, String unsafeExecutor2) { Objects.requireNonNull(unsafeExecutor); + Objects.requireNonNull(unsafeExecutor2); this.unsafeExecutor = unsafeExecutor; this.unsafeExecutor2 = unsafeExecutor2; } @Override - boolean allowedExecutors(Thread thread1, Thread thread2) { - return super.allowedExecutors(thread1, thread2) - || unsafeExecutor.equals(EsExecutors.executorName(thread1)) - || unsafeExecutor2 == null - || unsafeExecutor2.equals(EsExecutors.executorName(thread1)); + boolean allowedExecutors(Thread blockedThread, Thread completingThread) { + return super.allowedExecutors(blockedThread, completingThread) + || unsafeExecutor.equals(EsExecutors.executorName(blockedThread)) + || unsafeExecutor2.equals(EsExecutors.executorName(blockedThread)); } } diff --git a/server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java b/server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java new file mode 100644 index 0000000000000..68d934eb43bf3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class UnsafePlainActionFutureTests extends ESTestCase { + + public void testOneArg() { + String unsafeExecutorName = "unsafe-executor"; + String otherExecutorName = "other-executor"; + UnsafePlainActionFuture future = new UnsafePlainActionFuture(unsafeExecutorName); + Thread other1 = getThread(otherExecutorName); + Thread other2 = getThread(otherExecutorName); + assertFalse(future.allowedExecutors(other1, other2)); + Thread unsafe1 = getThread(unsafeExecutorName); + Thread unsafe2 = getThread(unsafeExecutorName); + assertTrue(future.allowedExecutors(unsafe1, unsafe2)); + + assertTrue(future.allowedExecutors(unsafe1, other1)); + } + + public void testTwoArg() { + String unsafeExecutorName1 = "unsafe-executor-1"; + String unsafeExecutorName2 = "unsafe-executor-2"; + String otherExecutorName = "other-executor"; + UnsafePlainActionFuture future = new UnsafePlainActionFuture(unsafeExecutorName1, unsafeExecutorName2); + Thread other1 = getThread(otherExecutorName); + Thread other2 = getThread(otherExecutorName); + assertFalse(future.allowedExecutors(other1, other2)); + Thread unsafe1Thread1 = getThread(unsafeExecutorName1); + Thread unsafe2Thread1 = getThread(unsafeExecutorName2); + Thread unsafe1Thread2 = getThread(unsafeExecutorName1); + Thread unsafe2Thread2 = getThread(unsafeExecutorName2); + assertTrue(future.allowedExecutors(unsafe1Thread1, unsafe1Thread2)); + assertTrue(future.allowedExecutors(unsafe2Thread1, unsafe2Thread2)); + + assertTrue(future.allowedExecutors(unsafe1Thread1, unsafe2Thread2)); + assertTrue(future.allowedExecutors(unsafe2Thread1, other1)); + assertTrue(future.allowedExecutors(other1, unsafe2Thread2)); + } + + private static Thread getThread(String executorName) { + Thread t = new Thread("[" + executorName + "][]"); + assertThat(EsExecutors.executorName(t), equalTo(executorName)); + return t; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java b/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java index 0264920c9d017..e6729a20d93e3 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java @@ -13,7 +13,7 @@ */ public class TestPlainActionFuture extends PlainActionFuture { @Override - boolean allowedExecutors(Thread thread1, Thread thread2) { + boolean allowedExecutors(Thread blockedThread, Thread completingThread) { return true; } } From f02be0779fc01d5de4b551ff62aaf8b75988710f Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Thu, 8 Aug 2024 22:25:14 +1000 Subject: [PATCH 04/37] Log all cause exception when authc fails (#111534) In RealmsAuthenticator, if a realm fails to authenticate the token, log the whole chain of exception causes. --- .../xpack/security/authc/RealmsAuthenticator.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java index 51af3a7eda665..c1995d0a26c5c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java @@ -284,7 +284,16 @@ private static void consumeNullUser( ) { messages.forEach((realm, tuple) -> { final String message = tuple.v1(); - final String cause = tuple.v2() == null ? "" : " (Caused by " + tuple.v2() + ")"; + Throwable ex = tuple.v2(); + var cause = new StringBuilder(); + while (ex != null) { + cause.append(cause.isEmpty() ? " (" : "; "); + cause.append("Caused by ").append(ex); + ex = ex.getCause(); + } + if (cause.isEmpty() == false) { + cause.append(')'); + } logger.warn("Authentication to realm {} failed - {}{}", realm.name(), message, cause); }); if (context.getUnlicensedRealms().isEmpty() == false) { From c6a7537df761ec02d2c3097029ae715cf59fcd6e Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 8 Aug 2024 09:23:56 -0500 Subject: [PATCH 05/37] Ingest download databases docs (#111688) Co-authored-by: Joe Gallo --- .../apis/delete-geoip-database.asciidoc | 55 +++++++++++++ .../ingest/apis/delete-pipeline.asciidoc | 2 +- ...tats-api.asciidoc => geoip-stats.asciidoc} | 0 .../ingest/apis/get-geoip-database.asciidoc | 80 +++++++++++++++++++ .../ingest/apis/get-pipeline.asciidoc | 2 +- docs/reference/ingest/apis/index.asciidoc | 23 +++++- .../ingest/apis/put-geoip-database.asciidoc | 72 +++++++++++++++++ .../ingest/processors/geoip.asciidoc | 7 +- 8 files changed, 233 insertions(+), 8 deletions(-) create mode 100644 docs/reference/ingest/apis/delete-geoip-database.asciidoc rename docs/reference/ingest/apis/{geoip-stats-api.asciidoc => geoip-stats.asciidoc} (100%) create mode 100644 docs/reference/ingest/apis/get-geoip-database.asciidoc create mode 100644 docs/reference/ingest/apis/put-geoip-database.asciidoc diff --git a/docs/reference/ingest/apis/delete-geoip-database.asciidoc b/docs/reference/ingest/apis/delete-geoip-database.asciidoc new file mode 100644 index 0000000000000..957e59f0f0de4 --- /dev/null +++ b/docs/reference/ingest/apis/delete-geoip-database.asciidoc @@ -0,0 +1,55 @@ +[[delete-geoip-database-api]] +=== Delete geoip database configuration API +++++ +Delete geoip database configuration +++++ + +Deletes a geoip database configuration. + +[source,console] +---- +DELETE /_ingest/geoip/database/my-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + + +[[delete-geoip-database-api-request]] +==== {api-request-title} + +`DELETE /_ingest/geoip/database/` + +[[delete-geoip-database-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the +`manage` <> to use this API. + +[[delete-geoip-database-api-path-params]] +==== {api-path-parms-title} + +``:: ++ +-- +(Required, string) Database configuration ID used to limit the request. + +-- + + +[[delete-geoip-database-api-query-params]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] + + +[[delete-geoip-database-api-example]] +==== {api-examples-title} + + +[[delete-geoip-database-api-specific-ex]] +===== Delete a specific geoip database configuration + +[source,console] +---- +DELETE /_ingest/geoip/database/example-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] diff --git a/docs/reference/ingest/apis/delete-pipeline.asciidoc b/docs/reference/ingest/apis/delete-pipeline.asciidoc index 6f50251dbf1cd..94ac87c61b56b 100644 --- a/docs/reference/ingest/apis/delete-pipeline.asciidoc +++ b/docs/reference/ingest/apis/delete-pipeline.asciidoc @@ -62,7 +62,7 @@ use a value of `*`. include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] -[[delete-pipeline-api-api-example]] +[[delete-pipeline-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/geoip-stats-api.asciidoc b/docs/reference/ingest/apis/geoip-stats.asciidoc similarity index 100% rename from docs/reference/ingest/apis/geoip-stats-api.asciidoc rename to docs/reference/ingest/apis/geoip-stats.asciidoc diff --git a/docs/reference/ingest/apis/get-geoip-database.asciidoc b/docs/reference/ingest/apis/get-geoip-database.asciidoc new file mode 100644 index 0000000000000..f055e3e759db8 --- /dev/null +++ b/docs/reference/ingest/apis/get-geoip-database.asciidoc @@ -0,0 +1,80 @@ +[[get-geoip-database-api]] +=== Get geoip database configuration API +++++ +Get geoip database configuration +++++ + +Returns information about one or more geoip database configurations. + +[source,console] +---- +GET /_ingest/geoip/database/my-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + + + +[[get-geoip-database-api-request]] +==== {api-request-title} + +`GET /_ingest/geoip/database/` + +`GET /_ingest/geoip/database` + +[[get-geoip-database-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the + `manage` <> to use this API. + +[[get-geoip-database-api-path-params]] +==== {api-path-parms-title} + +``:: +(Optional, string) +Comma-separated list of database configuration IDs to retrieve. Wildcard (`*`) expressions are +supported. ++ +To get all database configurations, omit this parameter or use `*`. + + +[[get-geoip-database-api-query-params]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + + +[[get-geoip-database-api-example]] +==== {api-examples-title} + + +[[get-geoip-database-api-specific-ex]] +===== Get information for a specific geoip database configuration + +[source,console] +---- +GET /_ingest/geoip/database/my-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + +The API returns the following response: + +[source,console-result] +---- +{ + "databases" : [ + { + "id" : "my-database-id", + "version" : 1, + "modified_date_millis" : 1723040276114, + "database" : { + "name" : "GeoIP2-Domain", + "maxmind" : { + "account_id" : "1234567" + } + } + } + ] +} +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc index 71a261d97bdeb..f2a1155bca12b 100644 --- a/docs/reference/ingest/apis/get-pipeline.asciidoc +++ b/docs/reference/ingest/apis/get-pipeline.asciidoc @@ -65,7 +65,7 @@ To get all ingest pipelines, omit this parameter or use `*`. include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -[[get-pipeline-api-api-example]] +[[get-pipeline-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/index.asciidoc b/docs/reference/ingest/apis/index.asciidoc index 04fcd500a9721..e068f99ea0ad3 100644 --- a/docs/reference/ingest/apis/index.asciidoc +++ b/docs/reference/ingest/apis/index.asciidoc @@ -13,7 +13,7 @@ Use the following APIs to create, manage, and test ingest pipelines: * <> to create or update a pipeline * <> to retrieve a pipeline configuration * <> to delete a pipeline -* <> to test a pipeline +* <> and <> to test ingest pipelines [discrete] [[ingest-stat-apis]] @@ -21,12 +21,27 @@ Use the following APIs to create, manage, and test ingest pipelines: Use the following APIs to get statistics about ingest processing: -* <> to get download statistics for GeoIP2 databases used with +* <> to get download statistics for IP geolocation databases used with the <>. +[discrete] +[[ingest-geoip-database-apis]] +=== Ingest GeoIP Database APIs + +preview::["The commercial IP geolocation database download management APIs are in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."] + +Use the following APIs to configure and manage commercial IP geolocation database downloads: + +* <> to create or update a database configuration +* <> to retrieve a database configuration +* <> to delete a database configuration + include::put-pipeline.asciidoc[] -include::delete-pipeline.asciidoc[] -include::geoip-stats-api.asciidoc[] include::get-pipeline.asciidoc[] +include::delete-pipeline.asciidoc[] include::simulate-pipeline.asciidoc[] include::simulate-ingest.asciidoc[] +include::geoip-stats.asciidoc[] +include::put-geoip-database.asciidoc[] +include::get-geoip-database.asciidoc[] +include::delete-geoip-database.asciidoc[] diff --git a/docs/reference/ingest/apis/put-geoip-database.asciidoc b/docs/reference/ingest/apis/put-geoip-database.asciidoc new file mode 100644 index 0000000000000..311c303002387 --- /dev/null +++ b/docs/reference/ingest/apis/put-geoip-database.asciidoc @@ -0,0 +1,72 @@ +[[put-geoip-database-api]] +=== Create or update geoip database configuration API +++++ +Create or update geoip database configuration +++++ + +Creates or updates an IP geolocation database configuration. + +IMPORTANT: The Maxmind `account_id` shown below requires a license key. Because the license key is sensitive information, +it is stored as a <> in {es} named `ingest.geoip.downloader.maxmind.license_key`. Only +one Maxmind license key is currently allowed per {es} cluster. A valid license key must be in the secure settings in order +to download from Maxmind. The license key setting does not take effect until all nodes are restarted. + +[source,console] +---- +PUT _ingest/geoip/database/my-database-id +{ + "name": "GeoIP2-Domain", + "maxmind": { + "account_id": "1025402" + } +} +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + +[[put-geoip-database-api-request]] +==== {api-request-title} + +`PUT /_ingest/geoip/database/` + +[[put-geoip-database-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the +`manage` <> to use this API. + + +[[put-geoip-database-api-path-params]] +==== {api-path-parms-title} + +``:: ++ +__ +(Required, string) ID of the database configuration to create or update. + +[[put-geoip-database-api-query-params]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] + +[[put-geoip-database-api-request-body]] +==== {api-request-body-title} + +// tag::geoip-database-object[] +`name`:: +(Required, string) +The provider-assigned name of the IP geolocation database to download. + +``:: +(Required, a provider object and its associated configuration) +The configuration necessary to identify which IP geolocation provider to use to download +the database, as well as any provider-specific configuration necessary for such downloading. ++ +At present, the only supported provider is `maxmind`, and the maxmind provider +requires that an `account_id` (string) is configured. +// end::geoip-database-object[] + +[[geoip-database-configuration-licensing]] +==== Licensing + +Downloading databases from third party providers is a commercial feature that requires an +appropriate license. For more information, refer to https://www.elastic.co/subscriptions. diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 230146d483144..3a9ba58dedbf0 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -24,6 +24,9 @@ stats API>>. If your cluster can't connect to the Elastic GeoIP endpoint or you want to manage your own updates, see <>. +If you would like to have {es} download database files directly from Maxmind using your own provided +license key, see <>. + If {es} can't connect to the endpoint for 30 days all updated databases will become invalid. {es} will stop enriching documents with geoip data and will add `tags: ["_geoip_expired_database"]` field instead. @@ -36,9 +39,9 @@ field instead. [options="header"] |====== | Name | Required | Default | Description -| `field` | yes | - | The field to get the ip address from for the geographical lookup. +| `field` | yes | - | The field to get the IP address from for the geographical lookup. | `target_field` | no | geoip | The field that will hold the geographical information looked up from the MaxMind database. -| `database_file` | no | GeoLite2-City.mmdb | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb) or the name of a supported database file in the `ingest-geoip` config directory. +| `database_file` | no | GeoLite2-City.mmdb | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb), or the name of a supported database file in the `ingest-geoip` config directory, or the name of a <> (with the `.mmdb` suffix appended). | `properties` | no | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document | `first_only` | no | `true` | If `true` only first found geoip data will be returned, even if `field` contains array From 414ac101b28c08b786840e1c2645508f74604eb6 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 8 Aug 2024 10:47:28 -0400 Subject: [PATCH 06/37] basic sorting tests (#111691) --- .../src/main/resources/date_nanos.csv-spec | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 63edfd355890a..b77689e1b5768 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -7,6 +7,24 @@ millis:date | nanos:date_nanos 2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z ; +sort by nanos +required_capability: date_nanos_type + +FROM date_nanos | SORT nanos DESC | KEEP millis, nanos | LIMIT 1; + +millis:date | nanos:date_nanos +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z +; + +sort by nanos asc +required_capability: date_nanos_type + +FROM date_nanos | WHERE millis > "2020-02-02" | SORT nanos ASC | KEEP millis, nanos | LIMIT 1; + +millis:date | nanos:date_nanos +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z +; + mv_max on date nanos required_capability: date_nanos_type From 0688cc27e83b90a6e0bfcc07d2a94aa697d5128c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 8 Aug 2024 18:27:30 +0300 Subject: [PATCH 07/37] Force using the last centroid during merging (#111644) * Force using the last centroid during merging * Update docs/changelog/111644.yaml * spotless * add asserts in test --- docs/changelog/111644.yaml | 6 ++++++ .../java/org/elasticsearch/tdigest/MergingDigest.java | 8 ++++++-- .../org/elasticsearch/tdigest/MergingDigestTests.java | 10 ++++++++++ .../java/org/elasticsearch/tdigest/TDigestTests.java | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/111644.yaml diff --git a/docs/changelog/111644.yaml b/docs/changelog/111644.yaml new file mode 100644 index 0000000000000..3705d697c95e3 --- /dev/null +++ b/docs/changelog/111644.yaml @@ -0,0 +1,6 @@ +pr: 111644 +summary: Force using the last centroid during merging +area: Aggregations +type: bug +issues: + - 111065 diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java index 0be2b68d76a21..172b0f24dfd99 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java @@ -92,7 +92,7 @@ public class MergingDigest extends AbstractTDigest { private final int[] order; // if true, alternate upward and downward merge passes - public boolean useAlternatingSort = true; + public boolean useAlternatingSort = false; // if true, use higher working value of compression during construction, then reduce on presentation public boolean useTwoLevelCompression = true; @@ -302,9 +302,13 @@ private void merge( addThis = projectedW <= wLimit; } if (i == 1 || i == incomingCount - 1) { - // force last centroid to never merge + // force first and last centroid to never merge addThis = false; } + if (lastUsedCell == mean.length - 1) { + // use the last centroid, there's no more + addThis = true; + } if (addThis) { // next point will fit diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java index 16a81bad50756..9fadf2218f203 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java @@ -151,4 +151,14 @@ public void testFill() { i++; } } + + public void testLargeInputSmallCompression() { + MergingDigest td = new MergingDigest(10); + for (int i = 0; i < 10_000_000; i++) { + td.add(between(0, 3_600_000)); + } + assertTrue(td.centroidCount() < 100); + assertTrue(td.quantile(0.00001) < 100_000); + assertTrue(td.quantile(0.99999) > 3_000_000); + } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java index 72b460da19da2..815346100532c 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java @@ -152,7 +152,7 @@ public void testQuantile() { hist2.compress(); double x1 = hist1.quantile(0.5); double x2 = hist2.quantile(0.5); - assertEquals(Dist.quantile(0.5, data), x1, 0.2); + assertEquals(Dist.quantile(0.5, data), x1, 0.25); assertEquals(x1, x2, 0.01); } From f90dc31c00c40b464f87aecf30e3fda361f4101f Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 8 Aug 2024 17:30:26 +0200 Subject: [PATCH 08/37] ESQL: Remove qualifier from attrs (#110581) In ES|QL's analyzer, we resolve names to field attributes, like some_field in FROM some_idx | KEEP some_field. The Attribute class, being copied from (S)QL, has a name and a qualified name, e.g. some_field and qualifier.some_field. However, we never use the qualifier, so remove it. --- .../xpack/esql/core/expression/Alias.java | 58 +++++---------- .../xpack/esql/core/expression/Attribute.java | 70 ++++-------------- .../esql/core/expression/EmptyAttribute.java | 12 +-- .../esql/core/expression/FieldAttribute.java | 74 ++++++++++++------- .../core/expression/MetadataAttribute.java | 62 +++++++++++----- .../core/expression/ReferenceAttribute.java | 48 ++++++++---- .../esql/core/expression/TypedAttribute.java | 12 +-- .../core/expression/UnresolvedAttribute.java | 37 +++------- .../esql/core/expression/UnresolvedStar.java | 2 +- .../expression/UnresolvedAttributeTests.java | 31 ++------ .../xpack/esql/analysis/Analyzer.java | 31 +++----- .../xpack/esql/analysis/AnalyzerRules.java | 31 ++------ .../function/UnsupportedAttribute.java | 16 +--- .../optimizer/LocalLogicalPlanOptimizer.java | 4 +- .../esql/optimizer/LogicalPlanOptimizer.java | 3 +- .../esql/optimizer/PhysicalPlanOptimizer.java | 2 +- .../ReplaceStatsAggExpressionWithEval.java | 9 +-- .../ReplaceStatsNestedExpressionWithEval.java | 2 +- .../optimizer/rules/SubstituteSurrogates.java | 4 +- .../rules/TranslateMetricsAggregate.java | 8 +- .../xpack/esql/parser/ExpressionBuilder.java | 4 +- .../xpack/esql/plan/logical/Enrich.java | 4 +- .../xpack/esql/plan/logical/Eval.java | 2 +- .../xpack/esql/plan/logical/InlineStats.java | 2 +- .../xpack/esql/plan/logical/join/Join.java | 2 +- .../esql/plugin/TransportEsqlQueryAction.java | 5 +- .../xpack/esql/session/EsqlSession.java | 12 +-- .../xpack/esql/expression/AliasTests.java | 13 ++-- .../function/FieldAttributeTests.java | 13 ++-- .../function/MetadataAttributeTests.java | 14 ++-- .../function/ReferenceAttributeTests.java | 13 ++-- .../esql/io/stream/PlanNamedTypesTests.java | 1 - .../optimizer/LogicalPlanOptimizerTests.java | 4 +- .../optimizer/PhysicalPlanOptimizerTests.java | 2 +- 34 files changed, 251 insertions(+), 356 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java index 01cc716a20547..2335a69d9ec88 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java @@ -16,7 +16,6 @@ import java.io.IOException; import java.util.List; -import java.util.Objects; import static java.util.Collections.singletonList; @@ -32,7 +31,6 @@ public final class Alias extends NamedExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(NamedExpression.class, "Alias", Alias::new); private final Expression child; - private final String qualifier; /** * Postpone attribute creation until it is actually created. @@ -41,21 +39,24 @@ public final class Alias extends NamedExpression { private Attribute lazyAttribute; public Alias(Source source, String name, Expression child) { - this(source, name, null, child, null); + this(source, name, child, null); } - public Alias(Source source, String name, String qualifier, Expression child) { - this(source, name, qualifier, child, null); + public Alias(Source source, String name, Expression child, NameId id) { + this(source, name, child, id, false); } - public Alias(Source source, String name, String qualifier, Expression child, NameId id) { - this(source, name, qualifier, child, id, false); - } - - public Alias(Source source, String name, String qualifier, Expression child, NameId id, boolean synthetic) { + public Alias(Source source, String name, Expression child, NameId id, boolean synthetic) { super(source, name, singletonList(child), id, synthetic); this.child = child; - this.qualifier = qualifier; + } + + @Deprecated + /** + * Old constructor from when this had a qualifier string. Still needed to not break serialization. + */ + private Alias(Source source, String name, String qualifier, Expression child, NameId id, boolean synthetic) { + this(source, name, child, id, synthetic); } public Alias(StreamInput in) throws IOException { @@ -73,7 +74,8 @@ public Alias(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeString(name()); - out.writeOptionalString(qualifier()); + // We used to write the qualifier here. We can still do if needed in the future. + out.writeOptionalString(null); out.writeNamedWriteable(child()); id().writeTo(out); out.writeBoolean(synthetic()); @@ -86,30 +88,22 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, Alias::new, name(), qualifier, child, id(), synthetic()); + return NodeInfo.create(this, Alias::new, name(), child, id(), synthetic()); } public Alias replaceChild(Expression child) { - return new Alias(source(), name(), qualifier, child, id(), synthetic()); + return new Alias(source(), name(), child, id(), synthetic()); } @Override public Alias replaceChildren(List newChildren) { - return new Alias(source(), name(), qualifier, newChildren.get(0), id(), synthetic()); + return new Alias(source(), name(), newChildren.get(0), id(), synthetic()); } public Expression child() { return child; } - public String qualifier() { - return qualifier; - } - - public String qualifiedName() { - return qualifier == null ? name() : qualifier + "." + name(); - } - @Override public Nullability nullable() { return child.nullable(); @@ -124,8 +118,8 @@ public DataType dataType() { public Attribute toAttribute() { if (lazyAttribute == null) { lazyAttribute = resolved() - ? new ReferenceAttribute(source(), name(), dataType(), qualifier, nullable(), id(), synthetic()) - : new UnresolvedAttribute(source(), name(), qualifier); + ? new ReferenceAttribute(source(), name(), dataType(), nullable(), id(), synthetic()) + : new UnresolvedAttribute(source(), name()); } return lazyAttribute; } @@ -146,18 +140,4 @@ public String nodeString() { public static Expression unwrap(Expression e) { return e instanceof Alias as ? as.child() : e; } - - @Override - public boolean equals(Object obj) { - if (super.equals(obj) == false) { - return false; - } - Alias other = (Alias) obj; - return Objects.equals(qualifier, other.qualifier); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), qualifier); - } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java index e89f39294a28b..3dda28a8abf9d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.core.expression; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -15,7 +14,6 @@ import java.util.Objects; import static java.util.Collections.emptyList; -import static org.elasticsearch.xpack.esql.core.util.StringUtils.splitQualifiedIndex; /** * {@link Expression}s that can be materialized and describe properties of the derived table. @@ -35,33 +33,19 @@ public static List getNamedWriteables() { return List.of(FieldAttribute.ENTRY, MetadataAttribute.ENTRY, ReferenceAttribute.ENTRY); } - // empty - such as a top level attribute in SELECT cause - // present - table name or a table name alias - private final String qualifier; - // cluster name in the qualifier (if any) - private final String cluster; - // can the attr be null - typically used in JOINs private final Nullability nullability; - public Attribute(Source source, String name, String qualifier, NameId id) { - this(source, name, qualifier, Nullability.TRUE, id); + public Attribute(Source source, String name, NameId id) { + this(source, name, Nullability.TRUE, id); } - public Attribute(Source source, String name, String qualifier, Nullability nullability, NameId id) { - this(source, name, qualifier, nullability, id, false); + public Attribute(Source source, String name, Nullability nullability, NameId id) { + this(source, name, nullability, id, false); } - public Attribute(Source source, String name, String qualifier, Nullability nullability, NameId id, boolean synthetic) { + public Attribute(Source source, String name, Nullability nullability, NameId id, boolean synthetic) { super(source, name, emptyList(), id, synthetic); - if (qualifier != null) { - Tuple splitQualifier = splitQualifiedIndex(qualifier); - this.cluster = splitQualifier.v1(); - this.qualifier = splitQualifier.v2(); - } else { - this.cluster = null; - this.qualifier = null; - } this.nullability = nullability; } @@ -70,14 +54,6 @@ public final Expression replaceChildren(List newChildren) { throw new UnsupportedOperationException("this type of node doesn't have any children to replace"); } - public String qualifier() { - return qualifier; - } - - public String qualifiedName() { - return qualifier == null ? name() : qualifier + "." + name(); - } - @Override public Nullability nullable() { return nullability; @@ -89,42 +65,26 @@ public AttributeSet references() { } public Attribute withLocation(Source source) { - return Objects.equals(source(), source) ? this : clone(source, name(), dataType(), qualifier(), nullable(), id(), synthetic()); - } - - public Attribute withQualifier(String qualifier) { - return Objects.equals(qualifier(), qualifier) - ? this - : clone(source(), name(), dataType(), qualifier, nullable(), id(), synthetic()); + return Objects.equals(source(), source) ? this : clone(source, name(), dataType(), nullable(), id(), synthetic()); } public Attribute withName(String name) { - return Objects.equals(name(), name) ? this : clone(source(), name, dataType(), qualifier(), nullable(), id(), synthetic()); + return Objects.equals(name(), name) ? this : clone(source(), name, dataType(), nullable(), id(), synthetic()); } public Attribute withNullability(Nullability nullability) { - return Objects.equals(nullable(), nullability) - ? this - : clone(source(), name(), dataType(), qualifier(), nullability, id(), synthetic()); + return Objects.equals(nullable(), nullability) ? this : clone(source(), name(), dataType(), nullability, id(), synthetic()); } public Attribute withId(NameId id) { - return clone(source(), name(), dataType(), qualifier(), nullable(), id, synthetic()); + return clone(source(), name(), dataType(), nullable(), id, synthetic()); } public Attribute withDataType(DataType type) { - return Objects.equals(dataType(), type) ? this : clone(source(), name(), type, qualifier(), nullable(), id(), synthetic()); + return Objects.equals(dataType(), type) ? this : clone(source(), name(), type, nullable(), id(), synthetic()); } - protected abstract Attribute clone( - Source source, - String name, - DataType type, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ); + protected abstract Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic); @Override public Attribute toAttribute() { @@ -143,19 +103,19 @@ public boolean semanticEquals(Expression other) { @Override protected Expression canonicalize() { - return clone(Source.EMPTY, name(), dataType(), qualifier, nullability, id(), synthetic()); + return clone(Source.EMPTY, name(), dataType(), nullability, id(), synthetic()); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), qualifier, nullability); + return Objects.hash(super.hashCode(), nullability); } @Override public boolean equals(Object obj) { if (super.equals(obj)) { Attribute other = (Attribute) obj; - return Objects.equals(qualifier, other.qualifier) && Objects.equals(nullability, other.nullability); + return Objects.equals(nullability, other.nullability); } return false; @@ -163,7 +123,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return qualifiedName() + "{" + label() + "}" + "#" + id(); + return name() + "{" + label() + "}" + "#" + id(); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java index 5824358e57525..d5c4be6e97c7e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java @@ -21,7 +21,7 @@ */ public class EmptyAttribute extends Attribute { public EmptyAttribute(Source source) { - super(source, StringUtils.EMPTY, null, null); + super(source, StringUtils.EMPTY, null); } @Override @@ -35,15 +35,7 @@ public String getWriteableName() { } @Override - protected Attribute clone( - Source source, - String name, - DataType type, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { + protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) { return this; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index 15578392c7f30..8e8973a11bc8a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -49,7 +49,7 @@ public FieldAttribute(Source source, String name, EsField field) { } public FieldAttribute(Source source, FieldAttribute parent, String name, EsField field) { - this(source, parent, name, field, null, Nullability.TRUE, null, false); + this(source, parent, name, field, Nullability.TRUE, null, false); } public FieldAttribute( @@ -57,12 +57,11 @@ public FieldAttribute( FieldAttribute parent, String name, EsField field, - String qualifier, Nullability nullability, NameId id, boolean synthetic ) { - this(source, parent, name, field.getDataType(), field, qualifier, nullability, id, synthetic); + this(source, parent, name, field.getDataType(), field, nullability, id, synthetic); } public FieldAttribute( @@ -71,17 +70,34 @@ public FieldAttribute( String name, DataType type, EsField field, - String qualifier, Nullability nullability, NameId id, boolean synthetic ) { - super(source, name, type, qualifier, nullability, id, synthetic); - this.path = parent != null ? parent.fieldName() : StringUtils.EMPTY; + super(source, name, type, nullability, id, synthetic); + this.path = parent != null ? parent.name() : StringUtils.EMPTY; this.parent = parent; this.field = field; } + @Deprecated + /** + * Old constructor from when this had a qualifier string. Still needed to not break serialization. + */ + private FieldAttribute( + Source source, + FieldAttribute parent, + String name, + DataType type, + EsField field, + String qualifier, + Nullability nullability, + NameId id, + boolean synthetic + ) { + this(source, parent, name, type, field, nullability, id, synthetic); + } + private FieldAttribute(StreamInput in) throws IOException { /* * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required @@ -112,7 +128,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(name()); dataType().writeTo(out); out.writeNamedWriteable(field); - out.writeOptionalString(qualifier()); + // We used to write the qualifier here. We can still do if needed in the future. + out.writeOptionalString(null); out.writeEnum(nullable()); id().writeTo(out); out.writeBoolean(synthetic()); @@ -130,7 +147,28 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, FieldAttribute::new, parent, name(), dataType(), field, qualifier(), nullable(), id(), synthetic()); + return NodeInfo.create( + this, + (source, parent1, name, type, field1, qualifier, nullability, id, synthetic) -> new FieldAttribute( + source, + parent1, + name, + type, + field1, + qualifier, + nullability, + id, + synthetic + ), + parent, + name(), + dataType(), + field, + (String) null, + nullable(), + id(), + synthetic() + ); } public FieldAttribute parent() { @@ -155,11 +193,6 @@ public String fieldName() { return Strings.hasText(path) ? path + "." + field.getName() : field.getName(); } - public String qualifiedPath() { - // return only the qualifier is there's no path - return qualifier() != null ? qualifier() + (Strings.hasText(path) ? "." + path : StringUtils.EMPTY) : path; - } - public EsField.Exact getExactInfo() { return field.getExactInfo(); } @@ -173,21 +206,12 @@ public FieldAttribute exactAttribute() { } private FieldAttribute innerField(EsField type) { - return new FieldAttribute(source(), this, name() + "." + type.getName(), type, qualifier(), nullable(), id(), synthetic()); + return new FieldAttribute(source(), this, name() + "." + type.getName(), type, nullable(), id(), synthetic()); } @Override - protected Attribute clone( - Source source, - String name, - DataType type, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { - FieldAttribute qualifiedParent = parent != null ? (FieldAttribute) parent.withQualifier(qualifier) : null; - return new FieldAttribute(source, qualifiedParent, name, field, qualifier, nullability, id, synthetic); + protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) { + return new FieldAttribute(source, parent, name, field, nullability, id, synthetic); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java index 7aa63d91aab09..539c55ba341cf 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java @@ -58,18 +58,34 @@ public MetadataAttribute( Source source, String name, DataType dataType, - String qualifier, Nullability nullability, NameId id, boolean synthetic, boolean searchable ) { - super(source, name, dataType, qualifier, nullability, id, synthetic); + super(source, name, dataType, nullability, id, synthetic); this.searchable = searchable; } public MetadataAttribute(Source source, String name, DataType dataType, boolean searchable) { - this(source, name, dataType, null, Nullability.TRUE, null, false, searchable); + this(source, name, dataType, Nullability.TRUE, null, false, searchable); + } + + @Deprecated + /** + * Old constructor from when this had a qualifier string. Still needed to not break serialization. + */ + private MetadataAttribute( + Source source, + String name, + DataType dataType, + String qualifier, + Nullability nullability, + NameId id, + boolean synthetic, + boolean searchable + ) { + this(source, name, dataType, nullability, id, synthetic, searchable); } @SuppressWarnings("unchecked") @@ -100,7 +116,8 @@ public void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeString(name()); dataType().writeTo(out); - out.writeOptionalString(qualifier()); + // We used to write the qualifier here. We can still do if needed in the future. + out.writeOptionalString(null); out.writeEnum(nullable()); id().writeTo(out); out.writeBoolean(synthetic()); @@ -118,16 +135,8 @@ public String getWriteableName() { } @Override - protected MetadataAttribute clone( - Source source, - String name, - DataType type, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { - return new MetadataAttribute(source, name, type, qualifier, nullability, id, synthetic, searchable); + protected MetadataAttribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) { + return new MetadataAttribute(source, name, type, null, nullability, id, synthetic, searchable); } @Override @@ -137,17 +146,32 @@ protected String label() { @Override protected NodeInfo info() { - return NodeInfo.create(this, MetadataAttribute::new, name(), dataType(), qualifier(), nullable(), id(), synthetic(), searchable); + return NodeInfo.create( + this, + (source, name, dataType, qualifier, nullability, id, synthetic, searchable1) -> new MetadataAttribute( + source, + name, + dataType, + qualifier, + nullability, + id, + synthetic, + searchable1 + ), + name(), + dataType(), + (String) null, + nullable(), + id(), + synthetic(), + searchable + ); } public boolean searchable() { return searchable; } - private MetadataAttribute withSource(Source source) { - return new MetadataAttribute(source, name(), dataType(), qualifier(), nullable(), id(), synthetic(), searchable()); - } - public static MetadataAttribute create(Source source, String name) { var t = ATTRIBUTES_MAP.get(name); return t != null ? new MetadataAttribute(source, name, t.v1(), t.v2()) : null; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java index 24bf95dcf660a..504e1eae8d880 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java @@ -28,10 +28,18 @@ public class ReferenceAttribute extends TypedAttribute { ); public ReferenceAttribute(Source source, String name, DataType dataType) { - this(source, name, dataType, null, Nullability.FALSE, null, false); + this(source, name, dataType, Nullability.FALSE, null, false); } - public ReferenceAttribute( + public ReferenceAttribute(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) { + super(source, name, dataType, nullability, id, synthetic); + } + + @Deprecated + /** + * Old constructor from when this had a qualifier string. Still needed to not break serialization. + */ + private ReferenceAttribute( Source source, String name, DataType dataType, @@ -40,7 +48,7 @@ public ReferenceAttribute( NameId id, boolean synthetic ) { - super(source, name, dataType, qualifier, nullability, id, synthetic); + this(source, name, dataType, nullability, id, synthetic); } @SuppressWarnings("unchecked") @@ -70,7 +78,8 @@ public void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeString(name()); dataType().writeTo(out); - out.writeOptionalString(qualifier()); + // We used to write the qualifier here. We can still do if needed in the future. + out.writeOptionalString(null); out.writeEnum(nullable()); id().writeTo(out); out.writeBoolean(synthetic()); @@ -87,21 +96,30 @@ public String getWriteableName() { } @Override - protected Attribute clone( - Source source, - String name, - DataType dataType, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { - return new ReferenceAttribute(source, name, dataType, qualifier, nullability, id, synthetic); + protected Attribute clone(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) { + return new ReferenceAttribute(source, name, dataType, null, nullability, id, synthetic); } @Override protected NodeInfo info() { - return NodeInfo.create(this, ReferenceAttribute::new, name(), dataType(), qualifier(), nullable(), id(), synthetic()); + return NodeInfo.create( + this, + (source, name, dataType, qualifier, nullability, id, synthetic) -> new ReferenceAttribute( + source, + name, + dataType, + qualifier, + nullability, + id, + synthetic + ), + name(), + dataType(), + (String) null, + nullable(), + id(), + synthetic() + ); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java index bf319856f9a93..0350abef99992 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java @@ -15,16 +15,8 @@ public abstract class TypedAttribute extends Attribute { private final DataType dataType; - protected TypedAttribute( - Source source, - String name, - DataType dataType, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { - super(source, name, qualifier, nullability, id, synthetic); + protected TypedAttribute(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) { + super(source, name, nullability, id, synthetic); this.dataType = dataType; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java index 87ef37cb84d1f..a6b519cc18525 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java @@ -28,26 +28,15 @@ public UnresolvedAttribute(Source source, String name) { this(source, name, null); } - public UnresolvedAttribute(Source source, String name, String qualifier) { - this(source, name, qualifier, null); - } - - public UnresolvedAttribute(Source source, String name, String qualifier, String unresolvedMessage) { - this(source, name, qualifier, null, unresolvedMessage, null); + public UnresolvedAttribute(Source source, String name, String unresolvedMessage) { + this(source, name, null, unresolvedMessage, null); } @SuppressWarnings("this-escape") - public UnresolvedAttribute( - Source source, - String name, - String qualifier, - NameId id, - String unresolvedMessage, - Object resolutionMetadata - ) { - super(source, name, qualifier, id); + public UnresolvedAttribute(Source source, String name, NameId id, String unresolvedMessage, Object resolutionMetadata) { + super(source, name, id); this.customMessage = unresolvedMessage != null; - this.unresolvedMsg = unresolvedMessage == null ? errorMessage(qualifiedName(), null) : unresolvedMessage; + this.unresolvedMsg = unresolvedMessage == null ? errorMessage(name(), null) : unresolvedMessage; this.resolutionMetadata = resolutionMetadata; } @@ -63,7 +52,7 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create(this, UnresolvedAttribute::new, name(), qualifier(), id(), unresolvedMsg, resolutionMetadata); + return NodeInfo.create(this, UnresolvedAttribute::new, name(), id(), unresolvedMsg, resolutionMetadata); } public Object resolutionMetadata() { @@ -80,20 +69,12 @@ public boolean resolved() { } @Override - protected Attribute clone( - Source source, - String name, - DataType dataType, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { + protected Attribute clone(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) { return this; } public UnresolvedAttribute withUnresolvedMessage(String unresolvedMessage) { - return new UnresolvedAttribute(source(), name(), qualifier(), id(), unresolvedMessage, resolutionMetadata()); + return new UnresolvedAttribute(source(), name(), id(), unresolvedMessage, resolutionMetadata()); } @Override @@ -103,7 +84,7 @@ public DataType dataType() { @Override public String toString() { - return UNRESOLVED_PREFIX + qualifiedName(); + return UNRESOLVED_PREFIX + name(); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java index f3b52cfcccf90..f4dd43cddf2d0 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java @@ -79,7 +79,7 @@ public boolean equals(Object obj) { } private String message() { - return (qualifier() != null ? qualifier().qualifiedName() + "." : "") + "*"; + return (qualifier() != null ? qualifier().name() + "." : "") + "*"; } @Override diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java index e7cd38b8f938a..6ac8377a2d8b9 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java @@ -18,11 +18,10 @@ public class UnresolvedAttributeTests extends AbstractNodeTestCase new UnresolvedAttribute( a.source(), randomValueOtherThan(a.name(), () -> randomAlphaOfLength(5)), - a.qualifier(), a.id(), a.unresolvedMessage(), a.resolutionMetadata() @@ -61,20 +59,11 @@ protected UnresolvedAttribute mutate(UnresolvedAttribute a) { () -> new UnresolvedAttribute( a.source(), a.name(), - randomValueOtherThan(a.qualifier(), UnresolvedAttributeTests::randomQualifier), - a.id(), - a.unresolvedMessage(), - a.resolutionMetadata() - ), - () -> new UnresolvedAttribute( - a.source(), - a.name(), - a.qualifier(), a.id(), randomValueOtherThan(a.unresolvedMessage(), () -> randomUnresolvedMessage()), a.resolutionMetadata() ), - () -> new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), a.unresolvedMessage(), new Object()) + () -> new UnresolvedAttribute(a.source(), a.name(), a.id(), a.unresolvedMessage(), new Object()) ) ); return option.get(); @@ -82,7 +71,7 @@ protected UnresolvedAttribute mutate(UnresolvedAttribute a) { @Override protected UnresolvedAttribute copy(UnresolvedAttribute a) { - return new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), a.unresolvedMessage(), a.resolutionMetadata()); + return new UnresolvedAttribute(a.source(), a.name(), a.id(), a.unresolvedMessage(), a.resolutionMetadata()); } @Override @@ -91,31 +80,25 @@ public void testTransform() { String newName = randomValueOtherThan(a.name(), () -> randomAlphaOfLength(5)); assertEquals( - new UnresolvedAttribute(a.source(), newName, a.qualifier(), a.id(), a.unresolvedMessage(), a.resolutionMetadata()), + new UnresolvedAttribute(a.source(), newName, a.id(), a.unresolvedMessage(), a.resolutionMetadata()), a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.name()) ? newName : v) ); - String newQualifier = randomValueOtherThan(a.qualifier(), UnresolvedAttributeTests::randomQualifier); - assertEquals( - new UnresolvedAttribute(a.source(), a.name(), newQualifier, a.id(), a.unresolvedMessage(), a.resolutionMetadata()), - a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.qualifier()) ? newQualifier : v) - ); - NameId newId = new NameId(); assertEquals( - new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), newId, a.unresolvedMessage(), a.resolutionMetadata()), + new UnresolvedAttribute(a.source(), a.name(), newId, a.unresolvedMessage(), a.resolutionMetadata()), a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.id()) ? newId : v) ); String newMessage = randomValueOtherThan(a.unresolvedMessage(), UnresolvedAttributeTests::randomUnresolvedMessage); assertEquals( - new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), newMessage, a.resolutionMetadata()), + new UnresolvedAttribute(a.source(), a.name(), a.id(), newMessage, a.resolutionMetadata()), a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.unresolvedMessage()) ? newMessage : v) ); Object newMeta = new Object(); assertEquals( - new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), a.unresolvedMessage(), newMeta), + new UnresolvedAttribute(a.source(), a.name(), a.id(), a.unresolvedMessage(), newMeta), a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.resolutionMetadata()) ? newMeta : v) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index abc9bf258caae..d4c258ece0134 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -125,7 +125,7 @@ public class Analyzer extends ParameterizedRuleExecutor NO_FIELDS = List.of( - new ReferenceAttribute(Source.EMPTY, "", DataType.NULL, null, Nullability.TRUE, null, true) + new ReferenceAttribute(Source.EMPTY, "", DataType.NULL, Nullability.TRUE, null, true) ); private static final Iterable> rules; @@ -218,7 +218,7 @@ protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) { private static List mappingAsAttributes(Source source, Map mapping) { var list = new ArrayList(); mappingAsAttributes(list, source, null, mapping); - list.sort(Comparator.comparing(Attribute::qualifiedName)); + list.sort(Comparator.comparing(Attribute::name)); return list; } @@ -286,7 +286,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) { ); } else { String error = context.enrichResolution().getError(policyName, plan.mode()); - var policyNameExp = new UnresolvedAttribute(plan.policyName().source(), policyName, null, error); + var policyNameExp = new UnresolvedAttribute(plan.policyName().source(), policyName, error); return new Enrich(plan.source(), plan.child(), plan.mode(), policyNameExp, plan.matchField(), null, Map.of(), List.of()); } } @@ -331,9 +331,9 @@ private static NamedExpression createEnrichFieldExpression( if (CollectionUtils.isEmpty(similar) == false) { msg += ", did you mean " + (similar.size() == 1 ? "[" + similar.get(0) + "]" : "any of " + similar) + "?"; } - return new UnresolvedAttribute(source, enrichFieldName, null, msg); + return new UnresolvedAttribute(source, enrichFieldName, msg); } else { - return new ReferenceAttribute(source, enrichFieldName, mappedField.dataType(), null, Nullability.TRUE, null, false); + return new ReferenceAttribute(source, enrichFieldName, mappedField.dataType(), Nullability.TRUE, null, false); } } } @@ -356,7 +356,7 @@ protected LogicalPlan rule(Lookup lookup, AnalyzerContext context) { if (CollectionUtils.isEmpty(potentialMatches) == false) { message = UnresolvedAttribute.errorMessage(tableName, potentialMatches).replace("column", "table"); } - tableNameExpression = new UnresolvedAttribute(tableNameExpression.source(), tableName, null, message); + tableNameExpression = new UnresolvedAttribute(tableNameExpression.source(), tableName, message); } // wrap the table in a local relationship for idiomatic field resolution else { @@ -498,15 +498,7 @@ private LogicalPlan resolveMvExpand(MvExpand p, List childrenOutput) p.child(), resolved, resolved.resolved() - ? new ReferenceAttribute( - resolved.source(), - resolved.name(), - resolved.dataType(), - null, - resolved.nullable(), - null, - false - ) + ? new ReferenceAttribute(resolved.source(), resolved.name(), resolved.dataType(), resolved.nullable(), null, false) : resolved ); } @@ -555,7 +547,6 @@ private LogicalPlan resolveLookup(Lookup l, List childrenOutput) { matchFieldChildReference = new UnresolvedAttribute( attr.source(), attr.name(), - attr.qualifier(), attr.id(), "column type mismatch, table column was [" + joinedAttribute.dataType().typeName() @@ -840,8 +831,8 @@ private DataType[] allowedEnrichTypes(String matchType) { } private static List resolveAgainstList(UnresolvedNamePattern up, Collection attrList) { - UnresolvedAttribute ua = new UnresolvedAttribute(up.source(), up.pattern(), null); - Predicate matcher = a -> up.match(a.name()) || up.match(a.qualifiedName()); + UnresolvedAttribute ua = new UnresolvedAttribute(up.source(), up.pattern()); + Predicate matcher = a -> up.match(a.name()); var matches = AnalyzerRules.maybeResolveAgainstList(matcher, () -> ua, attrList, true, a -> Analyzer.handleSpecialFields(ua, a)); return potentialCandidatesIfNoMatchesFound(ua, matches, attrList, list -> UnresolvedNamePattern.errorMessage(up.pattern(), list)); } @@ -1228,7 +1219,7 @@ private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap maybeResolveAgainstList( Collection attrList, java.util.function.Function fieldInspector ) { - // first take into account the qualified version - final String qualifier = u.qualifier(); final String name = u.name(); - final boolean qualified = u.qualifier() != null; - Predicate predicate = a -> { - return qualified ? Objects.equals(qualifier, a.qualifiedName()) : - // if the field is unqualified - // first check the names directly - (Objects.equals(name, a.name())) - // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields - || Objects.equals(name, a.qualifiedName()); - - }; + Predicate predicate = a -> Objects.equals(name, a.name()); return maybeResolveAgainstList(predicate, () -> u, attrList, false, fieldInspector); } @@ -117,7 +106,7 @@ public static List maybeResolveAgainstList( // found exact match or multiple if pattern if (matches.size() == 1 || isPattern) { // NB: only add the location if the match is univocal; b/c otherwise adding the location will overwrite any preexisting one - matches.replaceAll(e -> fieldInspector.apply(e)); + matches.replaceAll(fieldInspector::apply); return matches; } @@ -125,17 +114,9 @@ public static List maybeResolveAgainstList( List refs = matches.stream().sorted((a, b) -> { int lineDiff = a.sourceLocation().getLineNumber() - b.sourceLocation().getLineNumber(); int colDiff = a.sourceLocation().getColumnNumber() - b.sourceLocation().getColumnNumber(); - return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.qualifiedName().compareTo(b.qualifiedName())); - }) - .map( - a -> "line " - + a.sourceLocation().toString().substring(1) - + " [" - + (a.qualifier() != null ? "\"" + a.qualifier() + "\".\"" + a.name() + "\"" : a.name()) - + "]" - ) - .toList(); - - throw new IllegalStateException("Reference [" + ua.qualifiedName() + "] is ambiguous; " + "matches any of " + refs); + return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.name().compareTo(b.name())); + }).map(a -> "line " + a.sourceLocation().toString().substring(1) + " [" + a.name() + "]").toList(); + + throw new IllegalStateException("Reference [" + ua.name() + "] is ambiguous; " + "matches any of " + refs); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java index bf9c9eaa3c407..78577aa2b91e0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java @@ -66,9 +66,9 @@ public UnsupportedAttribute(Source source, String name, UnsupportedEsField field } public UnsupportedAttribute(Source source, String name, UnsupportedEsField field, String customMessage, NameId id) { - super(source, null, name, field, null, Nullability.TRUE, id, false); + super(source, null, name, field, Nullability.TRUE, id, false); this.hasCustomMessage = customMessage != null; - this.message = customMessage == null ? errorMessage(qualifiedName(), field) : customMessage; + this.message = customMessage == null ? errorMessage(name(), field) : customMessage; } private UnsupportedAttribute(StreamInput in) throws IOException { @@ -124,15 +124,7 @@ protected NodeInfo info() { } @Override - protected Attribute clone( - Source source, - String name, - DataType type, - String qualifier, - Nullability nullability, - NameId id, - boolean synthetic - ) { + protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) { return new UnsupportedAttribute(source, name, field(), hasCustomMessage ? message : null, id); } @@ -142,7 +134,7 @@ protected String label() { @Override public String toString() { - return "!" + qualifiedName(); + return "!" + name(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index edb8c0cde6052..69134923e6f47 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -153,14 +153,14 @@ else if (plan instanceof Project project) { Alias nullAlias = nullLiteral.get(f.dataType()); // save the first field as null (per datatype) if (nullAlias == null) { - Alias alias = new Alias(f.source(), f.name(), null, Literal.of(f, null), f.id()); + Alias alias = new Alias(f.source(), f.name(), Literal.of(f, null), f.id()); nullLiteral.put(dt, alias); projection = alias.toAttribute(); } // otherwise point to it else { // since avoids creating field copies - projection = new Alias(f.source(), f.name(), f.qualifier(), nullAlias.toAttribute(), f.id()); + projection = new Alias(f.source(), f.name(), nullAlias.toAttribute(), f.id()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index bad8080c3def4..fd24e6b1c519f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -336,7 +336,6 @@ public static > LogicalPlan pushGe new Alias( originalAttribute.source(), originalAttribute.name(), - originalAttribute.qualifier(), renamedAttribute, originalAttribute.id(), originalAttribute.synthetic() @@ -375,7 +374,7 @@ private static AttributeReplacement renameAttributesInExpressions( String tempName = locallyUniqueTemporaryName(a.name(), "temp_name"); // TODO: this should be synthetic // blocked on https://github.com/elastic/elasticsearch/issues/98703 - return new Alias(a.source(), tempName, null, a, null, false); + return new Alias(a.source(), tempName, a, null, false); }); return renamedAttribute.toAttribute(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java index 4237852551e8a..2a410d6f386a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java @@ -150,7 +150,7 @@ public PhysicalPlan apply(PhysicalPlan plan) { // add a synthetic field (so it doesn't clash with the user defined one) to return a constant // to avoid the block from being trimmed if (output.isEmpty()) { - var alias = new Alias(logicalFragment.source(), "", null, Literal.NULL, null, true); + var alias = new Alias(logicalFragment.source(), "", Literal.NULL, null, true); List fields = singletonList(alias); logicalFragment = new Eval(logicalFragment.source(), logicalFragment, fields); output = Expressions.asAttributes(fields); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java index 6cd61d03f1ffa..1746931f9a63e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java @@ -106,14 +106,7 @@ protected LogicalPlan rule(Aggregate aggregate) { Alias alias = rootAggs.get(canonical); if (alias == null) { // create synthetic alias ove the found agg function - alias = new Alias( - af.source(), - syntheticName(canonical, child, counter[0]++), - as.qualifier(), - canonical, - null, - true - ); + alias = new Alias(af.source(), syntheticName(canonical, child, counter[0]++), canonical, null, true); // and remember it to remove duplicates rootAggs.put(canonical, alias); // add it to the list of aggregates and continue diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java index 5bae99e1204f3..206bd6d3d1c76 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java @@ -103,7 +103,7 @@ protected LogicalPlan rule(Aggregate aggregate) { if (field instanceof Attribute == false && field.foldable() == false) { // 3. create a new alias if one doesn't exist yet no reference Attribute attr = expToAttribute.computeIfAbsent(field.canonical(), k -> { - Alias newAlias = new Alias(k.source(), syntheticName(k, af, counter[0]++), null, k, null, true); + Alias newAlias = new Alias(k.source(), syntheticName(k, af, counter[0]++), k, null, true); evals.add(newAlias); return newAlias.toAttribute(); }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java index fffda5622dd7e..4c5a04ff8353c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java @@ -82,7 +82,7 @@ protected LogicalPlan rule(Aggregate aggregate) { if (attr == null) { var temporaryName = LogicalPlanOptimizer.temporaryName(af, agg, counter[0]++); // create a synthetic alias (so it doesn't clash with a user defined name) - var newAlias = new Alias(agg.source(), temporaryName, null, af, null, true); + var newAlias = new Alias(agg.source(), temporaryName, af, null, true); attr = newAlias.toAttribute(); aggFuncToAttr.put(af, attr); newAggs.add(newAlias); @@ -92,7 +92,7 @@ protected LogicalPlan rule(Aggregate aggregate) { // 4. move the expression as an eval using the original alias // copy the original alias id so that other nodes using it down stream (e.g. eval referring to the original agg) // don't have to updated - var aliased = new Alias(agg.source(), agg.name(), null, surrogateWithRefs, agg.toAttribute().id()); + var aliased = new Alias(agg.source(), agg.name(), surrogateWithRefs, agg.toAttribute().id()); transientEval.add(aliased); } // the replacement is another aggregate function, so replace it in place diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java index 10c7a7325debc..f2926fe92ca3f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java @@ -147,12 +147,12 @@ LogicalPlan translate(Aggregate metrics) { return rateAgg.toAttribute(); }); if (changed.get()) { - secondPassAggs.add(new Alias(alias.source(), alias.name(), null, outerAgg, agg.id())); + secondPassAggs.add(new Alias(alias.source(), alias.name(), outerAgg, agg.id())); } else { var toPartial = new Alias(agg.source(), alias.name(), new ToPartial(agg.source(), af.field(), af)); var fromPartial = new FromPartial(agg.source(), toPartial.toAttribute(), af); firstPassAggs.add(toPartial); - secondPassAggs.add(new Alias(alias.source(), alias.name(), null, fromPartial, alias.id())); + secondPassAggs.add(new Alias(alias.source(), alias.name(), fromPartial, alias.id())); } } } @@ -200,10 +200,10 @@ LogicalPlan translate(Aggregate metrics) { newFinalGroup = timeBucket.toAttribute(); firstPassGroupings.add(newFinalGroup); } else { - newFinalGroup = new Alias(g.source(), g.name(), null, new Values(g.source(), g), g.id()); + newFinalGroup = new Alias(g.source(), g.name(), new Values(g.source(), g), g.id()); firstPassAggs.add(newFinalGroup); } - secondPassGroupings.add(new Alias(g.source(), g.name(), null, newFinalGroup.toAttribute(), g.id())); + secondPassGroupings.add(new Alias(g.source(), g.name(), newFinalGroup.toAttribute(), g.id())); } return newAggregate( newAggregate(metrics.child(), Aggregate.AggregateType.METRICS, firstPassAggs, firstPassGroupings), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 53281fae1b206..82b36f77ef69b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -657,7 +657,7 @@ public Alias visitField(EsqlBaseParser.FieldContext ctx) { UnresolvedAttribute id = visitQualifiedName(ctx.qualifiedName()); Expression value = expression(ctx.booleanExpression()); var source = source(ctx); - String name = id == null ? source.text() : id.qualifiedName(); + String name = id == null ? source.text() : id.name(); return new Alias(source, name, value); } @@ -688,7 +688,7 @@ public List visitGrouping(EsqlBaseParser.FieldsContext ctx) { name = source(field).text(); } } else { - name = id.qualifiedName(); + name = id.name(); } // wrap when necessary - no alias and no underlying attribute if (ne == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index 95c7ccb40de8f..e0e40b0242fa7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -235,9 +235,9 @@ public Enrich withGeneratedNames(List newNames) { if (enrichField.name().equals(newName)) { newEnrichFields.add(enrichField); } else if (enrichField instanceof ReferenceAttribute ra) { - newEnrichFields.add(new Alias(ra.source(), newName, ra.qualifier(), ra, new NameId(), ra.synthetic())); + newEnrichFields.add(new Alias(ra.source(), newName, ra, new NameId(), ra.synthetic())); } else if (enrichField instanceof Alias a) { - newEnrichFields.add(new Alias(a.source(), newName, a.qualifier(), a.child(), new NameId(), a.synthetic())); + newEnrichFields.add(new Alias(a.source(), newName, a.child(), new NameId(), a.synthetic())); } else { throw new IllegalArgumentException("Enrich field must be Alias or ReferenceAttribute"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index 10876211a95cf..2cecef42a42ac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -90,7 +90,7 @@ private List renameAliases(List originalAttributes, List n if (field.name().equals(newName)) { newFields.add(field); } else { - Alias newField = new Alias(field.source(), newName, field.qualifier(), field.child(), new NameId(), field.synthetic()); + Alias newField = new Alias(field.source(), newName, field.child(), new NameId(), field.synthetic()); newFields.add(newField); aliasReplacedByBuilder.put(field.toAttribute(), newField.toAttribute()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java index 0ad43f7029de0..187b3542e0607 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java @@ -183,7 +183,7 @@ private LogicalPlan ungroupedNextPhase(List schema, List firstP for (int i = 0; i < schema.size(); i++) { Attribute s = schema.get(i); Object value = BlockUtils.toJavaObject(p.getBlock(i), 0); - values.add(new Alias(source(), s.name(), null, new Literal(source(), value, s.dataType()), aggregates.get(i).id())); + values.add(new Alias(source(), s.name(), new Literal(source(), value, s.dataType()), aggregates.get(i).id())); } return new Eval(source(), child(), values); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 585d440c75147..7ad1bcad2d9d0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -156,7 +156,7 @@ public static List makeReference(List output) { List out = new ArrayList<>(output.size()); for (Attribute a : output) { if (a.resolved() && a instanceof ReferenceAttribute == false) { - out.add(new ReferenceAttribute(a.source(), a.name(), a.dataType(), a.qualifier(), a.nullable(), a.id(), a.synthetic())); + out.add(new ReferenceAttribute(a.source(), a.name(), a.dataType(), a.nullable(), a.id(), a.synthetic())); } else { out.add(a); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 9c23b343dd3b8..29adb7cd0b446 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -179,10 +179,7 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener columns = result.schema() - .stream() - .map(c -> new ColumnInfoImpl(c.qualifiedName(), c.dataType().outputType())) - .toList(); + List columns = result.schema().stream().map(c -> new ColumnInfoImpl(c.name(), c.dataType().outputType())).toList(); EsqlQueryResponse.Profile profile = configuration.profile() ? new EsqlQueryResponse.Profile(result.profiles()) : null; if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) { String id = asyncTask.getExecutionId().getEncoded(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index fc84a833923ba..a6bc7befccc80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -284,7 +284,7 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF if (p instanceof RegexExtract re) { // for Grok and Dissect // remove other down-the-tree references to the extracted fields for (Attribute extracted : re.extractedFields()) { - references.removeIf(attr -> matchByName(attr, extracted.qualifiedName(), false)); + references.removeIf(attr -> matchByName(attr, extracted.name(), false)); } // but keep the inputs needed by Grok/Dissect references.addAll(re.input().references()); @@ -316,16 +316,16 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF p.forEachExpressionDown(Alias.class, alias -> { // do not remove the UnresolvedAttribute that has the same name as its alias, ie "rename id = id" // or the UnresolvedAttributes that are used in Functions that have aliases "STATS id = MAX(id)" - if (p.references().names().contains(alias.qualifiedName())) { + if (p.references().names().contains(alias.name())) { return; } - references.removeIf(attr -> matchByName(attr, alias.qualifiedName(), keepCommandReferences.contains(attr))); + references.removeIf(attr -> matchByName(attr, alias.name(), keepCommandReferences.contains(attr))); }); }); // remove valid metadata attributes because they will be filtered out by the IndexResolver anyway // otherwise, in some edge cases, we will fail to ask for "*" (all fields) instead - references.removeIf(a -> a instanceof MetadataAttribute || MetadataAttribute.isSupported(a.qualifiedName())); + references.removeIf(a -> a instanceof MetadataAttribute || MetadataAttribute.isSupported(a.name())); Set fieldNames = references.names(); if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) { @@ -340,11 +340,11 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF } private static boolean matchByName(Attribute attr, String other, boolean skipIfPattern) { - boolean isPattern = Regex.isSimpleMatchPattern(attr.qualifiedName()); + boolean isPattern = Regex.isSimpleMatchPattern(attr.name()); if (skipIfPattern && isPattern) { return false; } - var name = attr.qualifiedName(); + var name = attr.name(); return isPattern ? Regex.simpleMatch(name, other) : name.equals(other); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java index 8bd4599bdb536..36f8b43e69378 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java @@ -35,11 +35,10 @@ public class AliasTests extends AbstractWireTestCase { public static Alias randomAlias() { Source source = SourceTests.randomSource(); String name = randomAlphaOfLength(5); - String qualifier = randomBoolean() ? null : randomAlphaOfLength(3); // TODO better randomChild Expression child = ReferenceAttributeTests.randomReferenceAttribute(); boolean synthetic = randomBoolean(); - return new Alias(source, name, qualifier, child, new NameId(), synthetic); + return new Alias(source, name, child, new NameId(), synthetic); } @Override @@ -51,16 +50,14 @@ protected Alias createTestInstance() { protected Alias mutateInstance(Alias instance) throws IOException { Source source = instance.source(); String name = instance.name(); - String qualifier = instance.qualifier(); Expression child = instance.child(); boolean synthetic = instance.synthetic(); - switch (between(0, 3)) { + switch (between(0, 2)) { case 0 -> name = randomAlphaOfLength(name.length() + 1); - case 1 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3)); - case 2 -> child = randomValueOtherThan(child, ReferenceAttributeTests::randomReferenceAttribute); - case 3 -> synthetic = false == synthetic; + case 1 -> child = randomValueOtherThan(child, ReferenceAttributeTests::randomReferenceAttribute); + case 2 -> synthetic = false == synthetic; } - return new Alias(source, name, qualifier, child, instance.id(), synthetic); + return new Alias(source, name, child, instance.id(), synthetic); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java index 7467dfb88d9a1..03befe66ac28e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java @@ -24,10 +24,9 @@ public static FieldAttribute createFieldAttribute(int maxDepth, boolean onlyRepr ? randomValueOtherThanMany(t -> false == DataType.isRepresentable(t), () -> randomFrom(DataType.types())) : randomFrom(DataType.types()); EsField field = AbstractEsFieldTypeTests.randomAnyEsField(maxDepth); - String qualifier = randomBoolean() ? null : randomAlphaOfLength(3); Nullability nullability = randomFrom(Nullability.values()); boolean synthetic = randomBoolean(); - return new FieldAttribute(source, parent, name, type, field, qualifier, nullability, new NameId(), synthetic); + return new FieldAttribute(source, parent, name, type, field, nullability, new NameId(), synthetic); } @Override @@ -42,18 +41,16 @@ protected FieldAttribute mutate(FieldAttribute instance) { String name = instance.name(); DataType type = instance.dataType(); EsField field = instance.field(); - String qualifier = instance.qualifier(); Nullability nullability = instance.nullable(); boolean synthetic = instance.synthetic(); - switch (between(0, 6)) { + switch (between(0, 5)) { case 0 -> parent = randomValueOtherThan(parent, () -> randomBoolean() ? null : createFieldAttribute(2, false)); case 1 -> name = randomAlphaOfLength(name.length() + 1); case 2 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types())); case 3 -> field = randomValueOtherThan(field, () -> AbstractEsFieldTypeTests.randomAnyEsField(3)); - case 4 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3)); - case 5 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values())); - case 6 -> synthetic = false == synthetic; + case 4 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values())); + case 5 -> synthetic = false == synthetic; } - return new FieldAttribute(source, parent, name, type, field, qualifier, nullability, new NameId(), synthetic); + return new FieldAttribute(source, parent, name, type, field, nullability, new NameId(), synthetic); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java index cf43a17361df5..dded8bbb478c1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java @@ -27,7 +27,7 @@ public static MetadataAttribute randomMetadataAttribute() { Nullability nullability = randomFrom(Nullability.values()); boolean synthetic = randomBoolean(); boolean searchable = randomBoolean(); - return new MetadataAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic, searchable); + return new MetadataAttribute(source, name, type, nullability, new NameId(), synthetic, searchable); } @Override @@ -35,18 +35,16 @@ protected MetadataAttribute mutate(MetadataAttribute instance) { Source source = instance.source(); String name = instance.name(); DataType type = instance.dataType(); - String qualifier = instance.qualifier(); Nullability nullability = instance.nullable(); boolean synthetic = instance.synthetic(); boolean searchable = instance.searchable(); - switch (between(0, 5)) { + switch (between(0, 4)) { case 0 -> name = randomAlphaOfLength(name.length() + 1); case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types())); - case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3)); - case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values())); - case 4 -> synthetic = false == synthetic; - case 5 -> searchable = false == searchable; + case 2 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values())); + case 3 -> synthetic = false == synthetic; + case 4 -> searchable = false == searchable; } - return new MetadataAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic, searchable); + return new MetadataAttribute(source, name, type, nullability, new NameId(), synthetic, searchable); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java index 31d1018bacc91..493cecffe8b3f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java @@ -18,10 +18,9 @@ public static ReferenceAttribute randomReferenceAttribute() { Source source = Source.EMPTY; String name = randomAlphaOfLength(5); DataType type = randomFrom(DataType.types()); - String qualifier = randomBoolean() ? null : randomAlphaOfLength(3); Nullability nullability = randomFrom(Nullability.values()); boolean synthetic = randomBoolean(); - return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic); + return new ReferenceAttribute(source, name, type, nullability, new NameId(), synthetic); } @Override @@ -34,16 +33,14 @@ protected ReferenceAttribute mutate(ReferenceAttribute instance) { Source source = instance.source(); String name = instance.name(); DataType type = instance.dataType(); - String qualifier = instance.qualifier(); Nullability nullability = instance.nullable(); boolean synthetic = instance.synthetic(); - switch (between(0, 4)) { + switch (between(0, 3)) { case 0 -> name = randomAlphaOfLength(name.length() + 1); case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types())); - case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3)); - case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values())); - case 4 -> synthetic = false == synthetic; + case 2 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values())); + case 3 -> synthetic = false == synthetic; } - return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic); + return new ReferenceAttribute(source, name, type, nullability, new NameId(), synthetic); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index b8259357849fe..a5f2adbc1fc29 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -209,7 +209,6 @@ static FieldAttribute randomFieldAttribute() { randomAlphaOfLength(randomIntBetween(1, 25)), // name randomDataType(), randomEsField(), - randomStringOrNull(), // qualifier randomNullability(), nameIdOrNull(), randomBoolean() // synthetic diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index be5d72e622512..c6b12eb0dc23f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -1771,7 +1771,7 @@ public void testDontPruneSameFieldDifferentDirectionSortClauses() { contains( new Order( EMPTY, - new ReferenceAttribute(EMPTY, "e", INTEGER, null, Nullability.TRUE, null, false), + new ReferenceAttribute(EMPTY, "e", INTEGER, Nullability.TRUE, null, false), Order.OrderDirection.ASC, Order.NullsPosition.LAST ), @@ -1814,7 +1814,7 @@ public void testPruneRedundantSortClauses() { contains( new Order( EMPTY, - new ReferenceAttribute(EMPTY, "e", INTEGER, null, Nullability.TRUE, null, false), + new ReferenceAttribute(EMPTY, "e", INTEGER, Nullability.TRUE, null, false), Order.OrderDirection.ASC, Order.NullsPosition.LAST ), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 7f8b90351b5b2..7dfa36011bf1f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -4548,7 +4548,7 @@ private static QueryBuilder findQueryBuilder(BoolQueryBuilder booleanQuery, Stri private void assertFieldExtractionWithDocValues(FieldExtractExec extract, DataType dataType, String... fieldNames) { extract.attributesToExtract().forEach(attr -> { - String name = attr.qualifiedName(); + String name = attr.name(); if (asList(fieldNames).contains(name)) { assertThat("Expected field '" + name + "' to use doc-values", extract.hasDocValuesAttribute(attr), equalTo(true)); assertThat("Expected field '" + name + "' to have data type " + dataType, attr.dataType(), equalTo(dataType)); From 9c6a2e03f69f5da76344253718402aa6bd85958a Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:04:37 -0400 Subject: [PATCH 09/37] [ES|QL] Combine Disjunctive CIDRMatch (#111501) * support CIDRMatch in CombineDisjunctions --- docs/changelog/111501.yaml | 6 + .../core/planner/ExpressionTranslators.java | 8 +- .../src/main/resources/ip.csv-spec | 72 +++++ .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../esql/optimizer/LogicalPlanOptimizer.java | 4 +- .../optimizer/rules/CombineDisjunctions.java | 155 ++++++++++ .../rules/CombineDisjunctionsToIn.java | 98 ------ .../LocalPhysicalPlanOptimizerTests.java | 51 ---- .../rules/CombineDisjunctionsTests.java | 281 ++++++++++++++++++ .../rules/CombineDisjunctionsToInTests.java | 132 -------- .../esql/planner/QueryTranslatorTests.java | 77 +++++ 11 files changed, 605 insertions(+), 286 deletions(-) create mode 100644 docs/changelog/111501.yaml create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java delete mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java diff --git a/docs/changelog/111501.yaml b/docs/changelog/111501.yaml new file mode 100644 index 0000000000000..a424142376e52 --- /dev/null +++ b/docs/changelog/111501.yaml @@ -0,0 +1,6 @@ +pr: 111501 +summary: "[ES|QL] Combine Disjunctive CIDRMatch" +area: ES|QL +type: enhancement +issues: + - 105143 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java index 2df3a8eba46d5..de7ea68f6201f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java @@ -260,8 +260,12 @@ private static Query boolQuery(Source source, Query left, Query right, boolean i } List queries; // check if either side is already a bool query to an extra bool query - if (left instanceof BoolQuery bool && bool.isAnd() == isAnd) { - queries = CollectionUtils.combine(bool.queries(), right); + if (left instanceof BoolQuery leftBool && leftBool.isAnd() == isAnd) { + if (right instanceof BoolQuery rightBool && rightBool.isAnd() == isAnd) { + queries = CollectionUtils.combine(leftBool.queries(), rightBool.queries()); + } else { + queries = CollectionUtils.combine(leftBool.queries(), right); + } } else if (right instanceof BoolQuery bool && bool.isAnd() == isAnd) { queries = CollectionUtils.combine(bool.queries(), left); } else { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index 697b1c899d65e..0fb6994ef759f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -282,6 +282,78 @@ str1:keyword |str2:keyword |ip1:ip |ip2:ip // end::to_ip-result[] ; +cdirMatchOrsIPs +required_capability: combine_disjunctive_cidrmatches + +FROM hosts +| WHERE CIDR_MATCH(ip1, "127.0.0.2/32") or CIDR_MATCH(ip0, "127.0.0.1") or CIDR_MATCH(ip1, "127.0.0.3/32") or CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") +| KEEP card, host, ip0, ip1 +| sort host, card, ip0, ip1 +; +warning:Line 2:20: evaluation of [ip1] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:20: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 2:55: evaluation of [ip0] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:55: java.lang.IllegalArgumentException: single-value function encountered multi-value + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth0 |alpha |127.0.0.1 |127.0.0.1 +eth0 |beta |127.0.0.1 |::1 +eth1 |beta |127.0.0.1 |127.0.0.2 +eth1 |beta |127.0.0.1 |128.0.0.1 +eth0 |gamma |fe80::cae2:65ff:fece:feb9|127.0.0.3 +lo0 |gamma |fe80::cae2:65ff:fece:feb9|fe81::cae2:65ff:fece:feb9 +; + +cdirMatchEqualsInsOrs +required_capability: combine_disjunctive_cidrmatches + +FROM hosts +| WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR CIDR_MATCH(ip0, "127.0.0.1") OR card IN ("eth0", "eth1") OR CIDR_MATCH(ip1, "127.0.0.3/32") OR card == "lo0" OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta" +| KEEP card, host, ip0, ip1 +| sort host, card, ip0, ip1 +; +warning:Line 2:58: evaluation of [ip1] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:58: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 2:93: evaluation of [ip0] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:93: java.lang.IllegalArgumentException: single-value function encountered multi-value + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth0 |alpha |127.0.0.1 |127.0.0.1 +eth1 |alpha |::1 |::1 +eth0 |beta |127.0.0.1 |::1 +eth1 |beta |127.0.0.1 |127.0.0.2 +eth1 |beta |127.0.0.1 |128.0.0.1 +eth0 |epsilon |[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:fec0, fe80::cae2:65ff:fece:fec1] |fe80::cae2:65ff:fece:fec1 +eth1 |epsilon |null |[127.0.0.1, 127.0.0.2, 127.0.0.3] +eth0 |gamma |fe80::cae2:65ff:fece:feb9 |127.0.0.3 +lo0 |gamma |fe80::cae2:65ff:fece:feb9 |fe81::cae2:65ff:fece:feb9 +; + +cdirMatchEqualsInsOrsIPs +required_capability: combine_disjunctive_cidrmatches + +FROM hosts +| WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR ip0 == "127.0.0.1" OR card IN ("eth0", "eth1") OR ip1 IN ("127.0.0.3", "127.0.0.1") OR card == "lo0" OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta" +| KEEP card, host, ip0, ip1 +| sort host, card, ip0, ip1 +; +warning:Line 2:58: evaluation of [ip1] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:58: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 2:82: evaluation of [ip0] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:82: java.lang.IllegalArgumentException: single-value function encountered multi-value + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth0 |alpha |127.0.0.1 |127.0.0.1 +eth1 |alpha |::1 |::1 +eth0 |beta |127.0.0.1 |::1 +eth1 |beta |127.0.0.1 |127.0.0.2 +eth1 |beta |127.0.0.1 |128.0.0.1 +eth0 |epsilon |[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:fec0, fe80::cae2:65ff:fece:fec1] |fe80::cae2:65ff:fece:fec1 +eth1 |epsilon |null |[127.0.0.1, 127.0.0.2, 127.0.0.3] +eth0 |gamma |fe80::cae2:65ff:fece:feb9 |127.0.0.3 +lo0 |gamma |fe80::cae2:65ff:fece:feb9 |fe81::cae2:65ff:fece:feb9 +; + pushDownIP from hosts | where ip1 == to_ip("::1") | keep card, host, ip0, ip1; ignoreOrder:true diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 50b2f3771e3b7..42eebffc0375b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -213,7 +213,12 @@ public enum Cap { /** * Support for nanosecond dates as a data type */ - DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG); + DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + + /** + * Support CIDRMatch in CombineDisjunctions rule. + */ + COMBINE_DISJUNCTIVE_CIDRMATCHES; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index fd24e6b1c519f..e55b090bbb35f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -28,7 +28,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.BooleanFunctionEqualsElimination; import org.elasticsearch.xpack.esql.optimizer.rules.BooleanSimplification; import org.elasticsearch.xpack.esql.optimizer.rules.CombineBinaryComparisons; -import org.elasticsearch.xpack.esql.optimizer.rules.CombineDisjunctionsToIn; +import org.elasticsearch.xpack.esql.optimizer.rules.CombineDisjunctions; import org.elasticsearch.xpack.esql.optimizer.rules.CombineEvals; import org.elasticsearch.xpack.esql.optimizer.rules.CombineProjections; import org.elasticsearch.xpack.esql.optimizer.rules.ConstantFolding; @@ -209,7 +209,7 @@ protected static Batch operators() { new PropagateNullable(), new BooleanFunctionEqualsElimination(), new CombineBinaryComparisons(), - new CombineDisjunctionsToIn(), + new CombineDisjunctions(), new SimplifyComparisonsArithmetics(DataType::areCompatible), // prune/elimination new PruneFilters(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java new file mode 100644 index 0000000000000..ceac1aa9ca75b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineOr; +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; + +/** + * Combine disjunctive Equals, In or CIDRMatch expressions on the same field into an In or CIDRMatch expression. + * This rule looks for both simple equalities: + * 1. a == 1 OR a == 2 becomes a IN (1, 2) + * and combinations of In + * 2. a == 1 OR a IN (2) becomes a IN (1, 2) + * 3. a IN (1) OR a IN (2) becomes a IN (1, 2) + * and combinations of CIDRMatch + * 4. CIDRMatch(a, ip1) OR CIDRMatch(a, ip2) OR a == ip3 or a IN (ip4, ip5) becomes CIDRMatch(a, ip1, ip2, ip3, ip4, ip5) + *

+ * This rule does NOT check for type compatibility as that phase has been + * already be verified in the analyzer. + */ +public final class CombineDisjunctions extends OptimizerRules.OptimizerExpressionRule { + public CombineDisjunctions() { + super(OptimizerRules.TransformDirection.UP); + } + + protected static In createIn(Expression key, List values, ZoneId zoneId) { + return new In(key.source(), key, values); + } + + protected static Equals createEquals(Expression k, Set v, ZoneId finalZoneId) { + return new Equals(k.source(), k, v.iterator().next(), finalZoneId); + } + + protected static CIDRMatch createCIDRMatch(Expression k, List v) { + return new CIDRMatch(k.source(), k, v); + } + + @Override + public Expression rule(Or or) { + Expression e = or; + // look only at equals, In and CIDRMatch + List exps = splitOr(e); + + Map> ins = new LinkedHashMap<>(); + Map> cidrs = new LinkedHashMap<>(); + Map> ips = new LinkedHashMap<>(); + ZoneId zoneId = null; + List ors = new LinkedList<>(); + boolean changed = false; + for (Expression exp : exps) { + if (exp instanceof Equals eq) { + // consider only equals against foldables + if (eq.right().foldable()) { + ins.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right()); + if (eq.left().dataType() == DataType.IP) { + Object value = eq.right().fold(); + // ImplicitCasting and ConstantFolding(includes explicit casting) are applied before CombineDisjunctions. + // They fold the input IP string to an internal IP format. These happen to Equals and IN, but not for CIDRMatch, + // as CIDRMatch takes strings as input, ImplicitCasting does not apply to it, and the first input to CIDRMatch is a + // field, ConstantFolding does not apply to it either. + // If the data type is IP, convert the internal IP format in Equals and IN to the format that is compatible with + // CIDRMatch, and store them in a separate map, so that they can be combined into existing CIDRMatch later. + if (value instanceof BytesRef bytesRef) { + value = ipToString(bytesRef); + } + ips.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(new Literal(Source.EMPTY, value, DataType.IP)); + } + } else { + ors.add(exp); + } + if (zoneId == null) { + zoneId = eq.zoneId(); + } + } else if (exp instanceof In in) { + ins.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list()); + if (in.value().dataType() == DataType.IP) { + List values = new ArrayList<>(in.list().size()); + for (Expression i : in.list()) { + Object value = i.fold(); + // Same as Equals. + if (value instanceof BytesRef bytesRef) { + value = ipToString(bytesRef); + } + values.add(new Literal(Source.EMPTY, value, DataType.IP)); + } + ips.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(values); + } + } else if (exp instanceof CIDRMatch cm) { + cidrs.computeIfAbsent(cm.ipField(), k -> new LinkedHashSet<>()).addAll(cm.matches()); + } else { + ors.add(exp); + } + } + + if (cidrs.isEmpty() == false) { + for (Expression f : ips.keySet()) { + cidrs.computeIfAbsent(f, k -> new LinkedHashSet<>()).addAll(ips.get(f)); + ins.remove(f); + } + } + + if (ins.isEmpty() == false) { + // combine equals alongside the existing ors + final ZoneId finalZoneId = zoneId; + ins.forEach( + (k, v) -> { ors.add(v.size() == 1 ? createEquals(k, v, finalZoneId) : createIn(k, new ArrayList<>(v), finalZoneId)); } + ); + + changed = true; + } + + if (cidrs.isEmpty() == false) { + cidrs.forEach((k, v) -> { ors.add(createCIDRMatch(k, new ArrayList<>(v))); }); + changed = true; + } + + if (changed) { + // TODO: this makes a QL `or`, not an ESQL `or` + Expression combineOr = combineOr(ors); + // check the result semantically since the result might different in order + // but be actually the same which can trigger a loop + // e.g. a == 1 OR a == 2 OR null --> null OR a in (1,2) --> literalsOnTheRight --> cycle + if (e.semanticEquals(combineOr) == false) { + e = combineOr; + } + } + + return e; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java deleted file mode 100644 index 42d4bf730a644..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.optimizer.rules; - -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; - -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineOr; -import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr; - -/** - * Combine disjunctions on the same field into an In expression. - * This rule looks for both simple equalities: - * 1. a == 1 OR a == 2 becomes a IN (1, 2) - * and combinations of In - * 2. a == 1 OR a IN (2) becomes a IN (1, 2) - * 3. a IN (1) OR a IN (2) becomes a IN (1, 2) - *

- * This rule does NOT check for type compatibility as that phase has been - * already be verified in the analyzer. - */ -public final class CombineDisjunctionsToIn extends OptimizerRules.OptimizerExpressionRule { - public CombineDisjunctionsToIn() { - super(OptimizerRules.TransformDirection.UP); - } - - protected In createIn(Expression key, List values, ZoneId zoneId) { - return new In(key.source(), key, values); - } - - protected Equals createEquals(Expression k, Set v, ZoneId finalZoneId) { - return new Equals(k.source(), k, v.iterator().next(), finalZoneId); - } - - @Override - public Expression rule(Or or) { - Expression e = or; - // look only at equals and In - List exps = splitOr(e); - - Map> found = new LinkedHashMap<>(); - ZoneId zoneId = null; - List ors = new LinkedList<>(); - - for (Expression exp : exps) { - if (exp instanceof Equals eq) { - // consider only equals against foldables - if (eq.right().foldable()) { - found.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right()); - } else { - ors.add(exp); - } - if (zoneId == null) { - zoneId = eq.zoneId(); - } - } else if (exp instanceof In in) { - found.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list()); - } else { - ors.add(exp); - } - } - - if (found.isEmpty() == false) { - // combine equals alongside the existing ors - final ZoneId finalZoneId = zoneId; - found.forEach( - (k, v) -> { ors.add(v.size() == 1 ? createEquals(k, v, finalZoneId) : createIn(k, new ArrayList<>(v), finalZoneId)); } - ); - - // TODO: this makes a QL `or`, not an ESQL `or` - Expression combineOr = combineOr(ors); - // check the result semantically since the result might different in order - // but be actually the same which can trigger a loop - // e.g. a == 1 OR a == 2 OR null --> null OR a in (1,2) --> literalsOnTheRight --> cycle - if (e.semanticEquals(combineOr) == false) { - e = combineOr; - } - } - - return e; - } -} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 614f6a31a88f9..5fba11c13561c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.Build; -import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.MapperService; @@ -64,15 +63,12 @@ import org.junit.Before; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import static java.util.Arrays.asList; -import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; @@ -670,44 +666,6 @@ public void testIsNotNull_TextField_Pushdown_WithCount() { assertThat(stat.query(), is(QueryBuilders.existsQuery("job"))); } - /** - * Expects - * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[],false] - * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, - * half_float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, - * unsigned_long{f}#16, version{f}#19, wildcard{f}#20]] - * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..][] - * \_EsQueryExec[test], query[{"esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.0/24"],"boost":1.0}},"source": - * "cidr_match(ip, \"127.0.0.0/24\")@1:19"}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[389] - */ - public void testCidrMatchPushdownFilter() { - var allTypeMappingAnalyzer = makeAnalyzer("mapping-ip.json", new EnrichResolution()); - final String fieldName = "ip_addr"; - - int cidrBlockCount = randomIntBetween(1, 10); - ArrayList cidrBlocks = new ArrayList<>(); - for (int i = 0; i < cidrBlockCount; i++) { - cidrBlocks.add(randomCidrBlock()); - } - String cidrBlocksString = cidrBlocks.stream().map((s) -> "\"" + s + "\"").collect(Collectors.joining(",")); - String cidrMatch = format(null, "cidr_match({}, {})", fieldName, cidrBlocksString); - - var query = "from test | where " + cidrMatch; - var plan = plannerOptimizer.plan(query, EsqlTestUtils.TEST_SEARCH_STATS, allTypeMappingAnalyzer); - - var limit = as(plan, LimitExec.class); - var exchange = as(limit.child(), ExchangeExec.class); - var project = as(exchange.child(), ProjectExec.class); - var field = as(project.child(), FieldExtractExec.class); - var queryExec = as(field.child(), EsQueryExec.class); - assertThat(queryExec.limit().fold(), is(1000)); - - var expectedInnerQuery = QueryBuilders.termsQuery(fieldName, cidrBlocks); - var expectedQuery = wrapWithSingleQuery(query, expectedInnerQuery, fieldName, new Source(1, 18, cidrMatch)); - assertThat(queryExec.query().toString(), is(expectedQuery.toString())); - } - private record OutOfRangeTestCase(String fieldName, String tooLow, String tooHigh) {}; public void testOutOfRangeFilterPushdown() { @@ -939,13 +897,4 @@ private Stat queryStatsFor(PhysicalPlan plan) { protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); } - - private String randomCidrBlock() { - boolean ipv4 = randomBoolean(); - - String address = NetworkAddress.format(randomIp(ipv4)); - int cidrPrefixLength = ipv4 ? randomIntBetween(0, 32) : randomIntBetween(0, 128); - - return format(null, "{}/{}", address, cidrPrefixLength); - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java new file mode 100644 index 0000000000000..2060327f1e18d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java @@ -0,0 +1,281 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.plan.logical.Filter; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.FOUR; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.SIX; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; +import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; +import static org.hamcrest.Matchers.contains; + +public class CombineDisjunctionsTests extends ESTestCase { + + public void testTwoEqualsWithOr() { + FieldAttribute fa = getFieldAttribute(); + + Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); + Expression e = new CombineDisjunctions().rule(or); + assertEquals(In.class, e.getClass()); + In in = (In) e; + assertEquals(fa, in.value()); + assertThat(in.list(), contains(ONE, TWO)); + } + + public void testTwoEqualsWithSameValue() { + FieldAttribute fa = getFieldAttribute(); + + Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE)); + Expression e = new CombineDisjunctions().rule(or); + assertEquals(Equals.class, e.getClass()); + Equals eq = (Equals) e; + assertEquals(fa, eq.left()); + assertEquals(ONE, eq.right()); + } + + public void testOneEqualsOneIn() { + FieldAttribute fa = getFieldAttribute(); + + Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO))); + Expression e = new CombineDisjunctions().rule(or); + assertEquals(In.class, e.getClass()); + In in = (In) e; + assertEquals(fa, in.value()); + assertThat(in.list(), contains(ONE, TWO)); + } + + public void testOneEqualsOneInWithSameValue() { + FieldAttribute fa = getFieldAttribute(); + + Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO))); + Expression e = new CombineDisjunctions().rule(or); + assertEquals(In.class, e.getClass()); + In in = (In) e; + assertEquals(fa, in.value()); + assertThat(in.list(), contains(ONE, TWO)); + } + + public void testSingleValueInToEquals() { + FieldAttribute fa = getFieldAttribute(); + + Equals equals = equalsOf(fa, ONE); + Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE))); + Expression e = new CombineDisjunctions().rule(or); + assertEquals(equals, e); + } + + public void testEqualsBehindAnd() { + FieldAttribute fa = getFieldAttribute(); + + And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); + Filter dummy = new Filter(EMPTY, relation(), and); + LogicalPlan transformed = new CombineDisjunctions().apply(dummy); + assertSame(dummy, transformed); + assertEquals(and, ((Filter) transformed).condition()); + } + + public void testTwoEqualsDifferentFields() { + FieldAttribute fieldOne = getFieldAttribute("ONE"); + FieldAttribute fieldTwo = getFieldAttribute("TWO"); + + Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO)); + Expression e = new CombineDisjunctions().rule(or); + assertEquals(or, e); + } + + public void testMultipleIn() { + FieldAttribute fa = getFieldAttribute(); + + Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO))); + Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); + Expression e = new CombineDisjunctions().rule(secondOr); + assertEquals(In.class, e.getClass()); + In in = (In) e; + assertEquals(fa, in.value()); + assertThat(in.list(), contains(ONE, TWO, THREE)); + } + + public void testOrWithNonCombinableExpressions() { + FieldAttribute fa = getFieldAttribute(); + + Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO)); + Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); + Expression e = new CombineDisjunctions().rule(secondOr); + assertEquals(Or.class, e.getClass()); + Or or = (Or) e; + assertEquals(or.left(), firstOr.right()); + assertEquals(In.class, or.right().getClass()); + In in = (In) or.right(); + assertEquals(fa, in.value()); + assertThat(in.list(), contains(ONE, THREE)); + } + + public void testCombineCIDRMatch() { + FieldAttribute faa = getFieldAttribute("a"); + FieldAttribute fab = getFieldAttribute("b"); + + List ipa1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + List ipa2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + List ipb1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + List ipb2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + + List ipa = new HashSet<>(CollectionUtils.combine(ipa1, ipa2)).stream().toList(); + List ipb = new HashSet<>(CollectionUtils.combine(ipb1, ipb2)).stream().toList(); + + List cidrs = new ArrayList<>(4); + cidrs.add(new CIDRMatch(EMPTY, faa, ipa1)); + cidrs.add(new CIDRMatch(EMPTY, fab, ipb1)); + cidrs.add(new CIDRMatch(EMPTY, faa, ipa2)); + cidrs.add(new CIDRMatch(EMPTY, fab, ipb2)); + Or oldOr = (Or) Predicates.combineOr(cidrs); + Expression e = new CombineDisjunctions().rule(oldOr); + assertEquals(Or.class, e.getClass()); + Or newOr = (Or) e; + assertEquals(CIDRMatch.class, newOr.left().getClass()); + assertEquals(CIDRMatch.class, newOr.right().getClass()); + + CIDRMatch cm1 = (CIDRMatch) newOr.left(); + CIDRMatch cm2 = (CIDRMatch) newOr.right(); + + if (cm1.ipField() == faa) { + assertEquals(fab, cm2.ipField()); + assertTrue(cm1.matches().size() == ipa.size() && cm1.matches().containsAll(ipa) && ipa.containsAll(cm1.matches())); + assertTrue(cm2.matches().size() == ipb.size() && cm2.matches().containsAll(ipb) && ipb.containsAll(cm2.matches())); + } else { + assertEquals(fab, cm1.ipField()); + assertTrue(cm1.matches().size() == ipb.size() && cm1.matches().containsAll(ipb) && ipb.containsAll(cm1.matches())); + assertTrue(cm2.matches().size() == ipa.size() && cm2.matches().containsAll(ipa) && ipa.containsAll(cm2.matches())); + } + } + + public void testCombineCIDRMatchEqualsIns() { + FieldAttribute fa_cidr_a = getFieldAttribute("cidr_a"); + FieldAttribute fa_cidr_b = getFieldAttribute("cidr_b"); + FieldAttribute fa_in_a = getFieldAttribute("in_a"); + FieldAttribute fa_eqin_a = getFieldAttribute("eqin_a"); + FieldAttribute fa_eq_a = getFieldAttribute("eq_a"); + + List ipa1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + List ipa2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + List ipb1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + List ipb2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP)); + + List ipa = new HashSet<>(CollectionUtils.combine(ipa1, ipa2)).stream().toList(); + List ipb = new HashSet<>(CollectionUtils.combine(ipb1, ipb2)).stream().toList(); + + List all = new ArrayList<>(10); + all.add(new CIDRMatch(EMPTY, fa_cidr_a, ipa1)); + all.add(new CIDRMatch(EMPTY, fa_cidr_b, ipb1)); + all.add(new CIDRMatch(EMPTY, fa_cidr_a, ipa2)); + all.add(new CIDRMatch(EMPTY, fa_cidr_b, ipb2)); + + all.add(new In(EMPTY, fa_in_a, List.of(ONE, TWO, THREE))); + all.add(new In(EMPTY, fa_eqin_a, List.of(FOUR, FIVE, SIX))); + all.add(new In(EMPTY, fa_in_a, List.of(THREE, FOUR, FIVE))); + + all.add(new Equals(EMPTY, fa_eq_a, ONE)); + all.add(new Equals(EMPTY, fa_eqin_a, ONE)); + all.add(new Equals(EMPTY, fa_eq_a, SIX)); + + Or oldOr = (Or) Predicates.combineOr(all); + + Expression e = new CombineDisjunctions().rule(oldOr); + assertEquals(Or.class, e.getClass()); + Or newOr = (Or) e; + assertEquals(Or.class, newOr.left().getClass()); + Or newOr1 = (Or) newOr.left(); + assertEquals(CIDRMatch.class, newOr.right().getClass()); + CIDRMatch cidr2 = (CIDRMatch) newOr.right(); + assertEquals(Or.class, e.getClass()); + + assertEquals(Or.class, newOr1.left().getClass()); + Or newOr2 = (Or) newOr1.left(); + assertEquals(In.class, newOr2.left().getClass()); + In in1 = (In) newOr2.left(); + assertEquals(In.class, newOr2.right().getClass()); + In in2 = (In) newOr2.right(); + assertEquals(Or.class, newOr1.right().getClass()); + Or newOr3 = (Or) newOr1.right(); + assertEquals(In.class, newOr3.left().getClass()); + In in3 = (In) newOr3.left(); + assertEquals(CIDRMatch.class, newOr3.right().getClass()); + CIDRMatch cidr1 = (CIDRMatch) newOr3.right(); + + if (cidr1.ipField() == fa_cidr_a) { + assertEquals(fa_cidr_b, cidr2.ipField()); + assertTrue(cidr1.matches().size() == ipa.size() && cidr1.matches().containsAll(ipa) && ipa.containsAll(cidr1.matches())); + assertTrue(cidr2.matches().size() == ipb.size() && cidr2.matches().containsAll(ipb) && ipb.containsAll(cidr2.matches())); + } else { + assertEquals(fa_cidr_b, cidr1.ipField()); + assertTrue(cidr1.matches().size() == ipb.size() && cidr1.matches().containsAll(ipb) && ipb.containsAll(cidr1.matches())); + assertTrue(cidr2.matches().size() == ipa.size() && cidr2.matches().containsAll(ipa) && ipa.containsAll(cidr2.matches())); + } + + if (in1.value() == fa_in_a) { + assertTrue(in1.list().size() == 5 && in1.list().containsAll(List.of(ONE, TWO, THREE, FOUR, FIVE))); + if (in2.value() == fa_eqin_a) { + assertEquals(in3.value(), fa_eq_a); + assertTrue(in3.list().size() == 2 && in3.list().containsAll(List.of(ONE, SIX))); + assertTrue(in2.list().size() == 4 && in2.list().containsAll(List.of(ONE, FOUR, FIVE, SIX))); + } else { + assertEquals(in2.value(), fa_eq_a); + assertTrue(in2.list().size() == 2 && in2.list().containsAll(List.of(ONE, SIX))); + assertTrue(in3.list().size() == 4 && in3.list().containsAll(List.of(ONE, FOUR, FIVE, SIX))); + } + } else if (in2.value() == fa_in_a) { + assertTrue(in2.list().size() == 5 && in2.list().containsAll(List.of(ONE, TWO, THREE, FOUR, FIVE))); + if (in1.value() == fa_eqin_a) { + assertEquals(in3.value(), fa_eq_a); + assertTrue(in3.list().size() == 2 && in3.list().containsAll(List.of(ONE, SIX))); + assertTrue(in1.list().size() == 4 && in1.list().containsAll(List.of(ONE, FOUR, FIVE, SIX))); + } else { + assertEquals(in3.value(), fa_eq_a); + assertTrue(in3.list().size() == 2 && in3.list().containsAll(List.of(ONE, SIX))); + assertTrue(in1.list().size() == 4 && in1.list().containsAll(List.of(ONE, FOUR, FIVE, SIX))); + } + } else { + assertTrue(in3.list().size() == 5 && in3.list().containsAll(List.of(ONE, TWO, THREE, FOUR, FIVE))); + if (in1.value() == fa_eqin_a) { + assertEquals(in2.value(), fa_eq_a); + assertTrue(in2.list().size() == 2 && in2.list().containsAll(List.of(ONE, SIX))); + assertTrue(in1.list().size() == 4 && in1.list().containsAll(List.of(ONE, FOUR, FIVE, SIX))); + } else { + assertEquals(in1.value(), fa_eq_a); + assertTrue(in1.list().size() == 2 && in1.list().containsAll(List.of(ONE, SIX))); + assertTrue(in2.list().size() == 4 && in2.list().containsAll(List.of(ONE, FOUR, FIVE, SIX))); + } + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java deleted file mode 100644 index 7bc2d69cb56e6..0000000000000 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.optimizer.rules; - -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.esql.plan.logical.Filter; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; - -import java.util.List; - -import static java.util.Arrays.asList; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; -import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; -import static org.hamcrest.Matchers.contains; - -public class CombineDisjunctionsToInTests extends ESTestCase { - public void testTwoEqualsWithOr() { - FieldAttribute fa = getFieldAttribute(); - - Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); - Expression e = new CombineDisjunctionsToIn().rule(or); - assertEquals(In.class, e.getClass()); - In in = (In) e; - assertEquals(fa, in.value()); - assertThat(in.list(), contains(ONE, TWO)); - } - - public void testTwoEqualsWithSameValue() { - FieldAttribute fa = getFieldAttribute(); - - Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE)); - Expression e = new CombineDisjunctionsToIn().rule(or); - assertEquals(Equals.class, e.getClass()); - Equals eq = (Equals) e; - assertEquals(fa, eq.left()); - assertEquals(ONE, eq.right()); - } - - public void testOneEqualsOneIn() { - FieldAttribute fa = getFieldAttribute(); - - Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO))); - Expression e = new CombineDisjunctionsToIn().rule(or); - assertEquals(In.class, e.getClass()); - In in = (In) e; - assertEquals(fa, in.value()); - assertThat(in.list(), contains(ONE, TWO)); - } - - public void testOneEqualsOneInWithSameValue() { - FieldAttribute fa = getFieldAttribute(); - - Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO))); - Expression e = new CombineDisjunctionsToIn().rule(or); - assertEquals(In.class, e.getClass()); - In in = (In) e; - assertEquals(fa, in.value()); - assertThat(in.list(), contains(ONE, TWO)); - } - - public void testSingleValueInToEquals() { - FieldAttribute fa = getFieldAttribute(); - - Equals equals = equalsOf(fa, ONE); - Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE))); - Expression e = new CombineDisjunctionsToIn().rule(or); - assertEquals(equals, e); - } - - public void testEqualsBehindAnd() { - FieldAttribute fa = getFieldAttribute(); - - And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); - Filter dummy = new Filter(EMPTY, relation(), and); - LogicalPlan transformed = new CombineDisjunctionsToIn().apply(dummy); - assertSame(dummy, transformed); - assertEquals(and, ((Filter) transformed).condition()); - } - - public void testTwoEqualsDifferentFields() { - FieldAttribute fieldOne = getFieldAttribute("ONE"); - FieldAttribute fieldTwo = getFieldAttribute("TWO"); - - Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO)); - Expression e = new CombineDisjunctionsToIn().rule(or); - assertEquals(or, e); - } - - public void testMultipleIn() { - FieldAttribute fa = getFieldAttribute(); - - Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO))); - Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); - Expression e = new CombineDisjunctionsToIn().rule(secondOr); - assertEquals(In.class, e.getClass()); - In in = (In) e; - assertEquals(fa, in.value()); - assertThat(in.list(), contains(ONE, TWO, THREE)); - } - - public void testOrWithNonCombinableExpressions() { - FieldAttribute fa = getFieldAttribute(); - - Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO)); - Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); - Expression e = new CombineDisjunctionsToIn().rule(secondOr); - assertEquals(Or.class, e.getClass()); - Or or = (Or) e; - assertEquals(or.left(), firstOr.right()); - assertEquals(In.class, or.right().getClass()); - In in = (In) or.right(); - assertEquals(fa, in.value()); - assertThat(in.list(), contains(ONE, THREE)); - } -} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 370e2af819f61..fee260cc7c657 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -31,10 +31,13 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.matchesRegex; +//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class QueryTranslatorTests extends ESTestCase { private static TestPlannerOptimizer plannerOptimizer; + private static TestPlannerOptimizer plannerOptimizerIPs; + private static Analyzer makeAnalyzer(String mappingFileName) { var mapping = loadMapping(mappingFileName); EsIndex test = new EsIndex("test", mapping, Set.of("test")); @@ -49,6 +52,7 @@ private static Analyzer makeAnalyzer(String mappingFileName) { @BeforeClass public static void init() { plannerOptimizer = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-all-types.json")); + plannerOptimizerIPs = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-hosts.json")); } @Override @@ -63,6 +67,13 @@ public void assertQueryTranslation(String query, Matcher translationMatc assertThat(translatedQuery, translationMatcher); } + public void assertQueryTranslationIPs(String query, Matcher translationMatcher) { + PhysicalPlan optimized = plannerOptimizerIPs.plan(query); + EsQueryExec eqe = (EsQueryExec) optimized.collectLeaves().get(0); + final String translatedQuery = eqe.query().toString().replaceAll("\\s+", ""); + assertThat(translatedQuery, translationMatcher); + } + public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE 10 < integer""", containsString(""" @@ -152,4 +163,70 @@ public void testRanges() { .*must.*esql_single_value":\\{"field":"unsigned_long\"""" + """ .*"range":\\{"unsigned_long":\\{"gte":2147483648,.*"range":\\{"unsigned_long":\\{"lte":2147483650,.*""")); } + + public void testIPs() { + // Nothing to combine + assertQueryTranslationIPs(""" + FROM hosts | WHERE CIDR_MATCH(ip0, "127.0.0.1") OR CIDR_MATCH(ip0, "127.0.0.3") AND CIDR_MATCH(ip1, "fe80::cae2:65ff:fece:fec0") + """, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1".*""" + """ + .*bool.*must.*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.3".*""" + """ + esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["fe80::cae2:65ff:fece:fec0".*""")); + + // Combine Equals, In and CIDRMatch on IP type + assertQueryTranslationIPs(""" + FROM hosts | WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR CIDR_MATCH(ip1, "127.0.0.3/32") \ + OR card IN ("eth0", "eth1") OR card == "lo0" OR CIDR_MATCH(ip0, "127.0.0.1") OR \ + CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """ + esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """ + esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.2/32","127.0.0.3/32".*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); + + assertQueryTranslationIPs(""" + FROM hosts | WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR CIDR_MATCH(ip1, "127.0.0.3/32") \ + OR card IN ("eth0", "eth1") OR card == "lo0" OR CIDR_MATCH(ip0, "127.0.0.1") OR \ + CIDR_MATCH(ip0, "127.0.0.0/24", "172.0.0.0/31") OR CIDR_MATCH(ip0, "127.0.1.0/24") OR \ + CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:fec0", "172.0.2.0/24") OR \ + CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """ + esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """ + esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.2/32","127.0.0.3/32".*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","127.0.0.0/24","172.0.0.0/31","127.0.1.0/24",\ + "fe80::cae2:65ff:fece:fec0","172.0.2.0/24","fe80::cae2:65ff:fece:feb9".*""")); + + assertQueryTranslationIPs(""" + FROM hosts | WHERE host == "alpha" OR host == "gamma" OR ip1 IN ("127.0.0.2"::ip) OR CIDR_MATCH(ip1, "127.0.0.3/32") \ + OR card IN ("eth0", "eth1") OR card == "lo0" OR ip0 IN ("127.0.0.1"::ip, "128.0.0.1"::ip) \ + OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """ + esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """ + esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); + + assertQueryTranslationIPs(""" + FROM hosts | WHERE host == "alpha" OR host == "gamma" OR ip1 == "127.0.0.2"::ip OR CIDR_MATCH(ip1, "127.0.0.3/32") \ + OR card IN ("eth0", "eth1") OR card == "lo0" OR ip0 IN ("127.0.0.1"::ip, "128.0.0.1"::ip) \ + OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """ + esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """ + esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); + + assertQueryTranslationIPs(""" + FROM hosts | WHERE host == "alpha" OR host == "gamma" OR ip1 == "127.0.0.2" OR CIDR_MATCH(ip1, "127.0.0.3/32") \ + OR card IN ("eth0", "eth1") OR card == "lo0" OR ip0 IN ("127.0.0.1"::ip, "128.0.0.1"::ip) \ + OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """ + esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """ + esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """ + esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); + } } From aaf65b744b06336394c8443e15cbb258d6b22e1e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 9 Aug 2024 03:10:39 +1000 Subject: [PATCH 10/37] Forward port release notes for v8.15.0 (#111714) --- .../reference/migration/migrate_8_15.asciidoc | 122 ++++- docs/reference/release-notes/8.15.0.asciidoc | 511 +++++++++++++++++- .../release-notes/highlights.asciidoc | 37 ++ 3 files changed, 660 insertions(+), 10 deletions(-) diff --git a/docs/reference/migration/migrate_8_15.asciidoc b/docs/reference/migration/migrate_8_15.asciidoc index a183e68a50693..1961230da1bbf 100644 --- a/docs/reference/migration/migrate_8_15.asciidoc +++ b/docs/reference/migration/migrate_8_15.asciidoc @@ -16,5 +16,125 @@ coming::[8.15.0] [[breaking-changes-8.15]] === Breaking changes -There are no breaking changes in {es} 8.15. +The following changes in {es} 8.15 might affect your applications +and prevent them from operating normally. +Before upgrading to 8.15, review these changes and take the described steps +to mitigate the impact. + +[discrete] +[[breaking_815_cluster_and_node_setting_changes]] +==== Cluster and node setting changes + +[[change_skip_unavailable_remote_cluster_setting_default_value_to_true]] +.Change `skip_unavailable` remote cluster setting default value to true +[%collapsible] +==== +*Details* + +The default value of the `skip_unavailable` setting is now set to true. All existing and future remote clusters that do not define this setting will use the new default. This setting only affects cross-cluster searches using the _search or _async_search API. + +*Impact* + +Unavailable remote clusters in a cross-cluster search will no longer cause the search to fail unless skip_unavailable is configured to be `false` in elasticsearch.yml or via the `_cluster/settings` API. Unavailable clusters with `skip_unavailable`=`true` (either explicitly or by using the new default) are marked as SKIPPED in the search response metadata section and do not fail the entire search. If users want to ensure that a search returns a failure when a particular remote cluster is not available, `skip_unavailable` must be now be set explicitly. +==== + +[discrete] +[[breaking_815_rollup_changes]] +==== Rollup changes + +[[disallow_new_rollup_jobs_in_clusters_with_no_rollup_usage]] +.Disallow new rollup jobs in clusters with no rollup usage +[%collapsible] +==== +*Details* + +The put rollup API will fail with an error when a rollup job is created in a cluster with no rollup usage + +*Impact* + +Clusters with no rollup usage (either no rollup job or index) can not create new rollup jobs +==== + +[discrete] +[[breaking_815_rest_api_changes]] +==== REST API changes + +[[interpret_timeout_1_as_infinite_ack_timeout]] +.Interpret `?timeout=-1` as infinite ack timeout +[%collapsible] +==== +*Details* + +Today {es} accepts the parameter `?timeout=-1` in many APIs, but interprets +this to mean the same as `?timeout=0`. From 8.15 onwards `?timeout=-1` will +mean to wait indefinitely, aligning the behaviour of this parameter with +other similar parameters such as `?master_timeout`. + +*Impact* + +Use `?timeout=0` to force relevant operations to time out immediately +instead of `?timeout=-1` +==== + +[[replace_model_id_with_inference_id]] +.Replace `model_id` with `inference_id` in GET inference API +[%collapsible] +==== +*Details* + +From 8.15 onwards the <> response will return an +`inference_id` field instead of a `model_id`. + +*Impact* + +If your application uses the `model_id` in a GET inference API response, +switch it to use `inference_id` instead. +==== + + +[discrete] +[[deprecated-8.15]] +=== Deprecations + +The following functionality has been deprecated in {es} 8.15 +and will be removed in a future version. +While this won't have an immediate impact on your applications, +we strongly encourage you to take the described steps to update your code +after upgrading to 8.15. + +To find out if you are using any deprecated functionality, +enable <>. + +[discrete] +[[deprecations_815_cluster_and_node_setting]] +==== Cluster and node setting deprecations + +[[deprecate_absolute_size_values_for_indices_breaker_total_limit_setting]] +.Deprecate absolute size values for `indices.breaker.total.limit` setting +[%collapsible] +==== +*Details* + +Previously, the value of `indices.breaker.total.limit` could be specified as an absolute size in bytes. This setting controls the overal amount of memory the server is allowed to use before taking remedial actions. Setting this to a specific number of bytes led to strange behaviour when the node maximum heap size changed because the circut breaker limit would remain unchanged. This would either leave the value too low, causing part of the heap to remain unused; or it would leave the value too high, causing the circuit breaker to be ineffective at preventing OOM errors. The only reasonable behaviour for this setting is that it scales with the size of the heap, and so absolute byte limits are now deprecated. + +*Impact* + +Users must change their configuration to specify a percentage instead of an absolute number of bytes for `indices.breaker.total.limit`, or else accept the default, which is already specified as a percentage. +==== + +[discrete] +[[deprecations_815_rest_api]] +==== REST API deprecations + +[[deprecate_text_expansion_weighted_tokens_queries]] +.Deprecate `text_expansion` and `weighted_tokens` queries +[%collapsible] +==== +*Details* + +The `text_expansion` and `weighted_tokens` queries have been replaced by `sparse_vector`. + +*Impact* + +Please update your existing `text_expansion` and `weighted_tokens` queries to use `sparse_vector.` +==== + +[[deprecate_using_slm_privileges_to_access_ilm]] +.Deprecate using slm privileges to access ilm +[%collapsible] +==== +*Details* + +The `read_slm` privilege can get the ILM status, and the `manage_slm` privilege can start and stop ILM. Access to these APIs should be granted using the `read_ilm` and `manage_ilm` privileges instead. Access to ILM APIs will be removed from SLM privileges in a future major release, and is now deprecated. + +*Impact* + +Users that need access to the ILM status API should now use the `read_ilm` privilege. Users that need to start and stop ILM, should use the `manage_ilm` privilege. +==== diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index c13c1c95c09ff..40d5cf02954bf 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -1,16 +1,509 @@ [[release-notes-8.15.0]] == {es} version 8.15.0 -coming[8.15.0] - Also see <>. -[[known-issues-8.15.0]] +[[breaking-8.15.0]] +[float] +=== Breaking changes + +Cluster Coordination:: +* Interpret `?timeout=-1` as infinite ack timeout {es-pull}107675[#107675] + +Inference API:: +* Replace `model_id` with `inference_id` in GET inference API {es-pull}111366[#111366] + +Rollup:: +* Disallow new rollup jobs in clusters with no rollup usage {es-pull}108624[#108624] (issue: {es-issue}108381[#108381]) + +Search:: +* Change `skip_unavailable` remote cluster setting default value to true {es-pull}105792[#105792] + +[[bug-8.15.0]] +[float] +=== Bug fixes + +Aggregations:: +* Don't sample calls to `ReduceContext#consumeBucketsAndMaybeBreak` ins `InternalDateHistogram` and `InternalHistogram` during reduction {es-pull}110186[#110186] +* Fix `ClassCastException` in Significant Terms {es-pull}108429[#108429] (issue: {es-issue}108427[#108427]) +* Run terms concurrently when cardinality is only lower than shard size {es-pull}110369[#110369] (issue: {es-issue}105505[#105505]) + +Allocation:: +* Fix misc trappy allocation API timeouts {es-pull}109241[#109241] +* Fix trappy timeout in allocation explain API {es-pull}109240[#109240] + +Analysis:: +* Correct positioning for unique token filter {es-pull}109395[#109395] + +Authentication:: +* Add comma before charset parameter in WWW-Authenticate response header {es-pull}110906[#110906] +* Avoid NPE if `users_roles` file does not exist {es-pull}109606[#109606] +* Improve security-crypto threadpool overflow handling {es-pull}111369[#111369] + +Authorization:: +* Fix trailing slash in `security.put_privileges` specification {es-pull}110177[#110177] +* Fixes cluster state-based role mappings not recovered from disk {es-pull}109167[#109167] +* Handle unmatching remote cluster wildcards properly for `IndicesRequest.SingleIndexNoWildcards` requests {es-pull}109185[#109185] + +Autoscaling:: +* Expose `?master_timeout` in autoscaling APIs {es-pull}108759[#108759] + +CRUD:: +* Update checkpoints after post-replication actions, even on failure {es-pull}109908[#109908] + +Cluster Coordination:: +* Deserialize publish requests on generic thread-pool {es-pull}108814[#108814] (issue: {es-issue}106352[#106352]) +* Fail cluster state API if blocked {es-pull}109618[#109618] (issue: {es-issue}107503[#107503]) +* Use `scheduleUnlessShuttingDown` in `LeaderChecker` {es-pull}108643[#108643] (issue: {es-issue}108642[#108642]) + +Data streams:: +* Apm-data: set concrete values for `metricset.interval` {es-pull}109043[#109043] +* Ecs@mappings: reduce scope for `ecs_geo_point` {es-pull}108349[#108349] (issue: {es-issue}108338[#108338]) +* Include component templates in retention validaiton {es-pull}109779[#109779] + +Distributed:: +* Associate restore snapshot task to parent mount task {es-pull}108705[#108705] (issue: {es-issue}105830[#105830]) +* Don't detect `PlainActionFuture` deadlock on concurrent complete {es-pull}110361[#110361] (issues: {es-issue}110181[#110181], {es-issue}110360[#110360]) +* Handle nullable `DocsStats` and `StoresStats` {es-pull}109196[#109196] + +Downsampling:: +* Support flattened fields and multi-fields as dimensions in downsampling {es-pull}110066[#110066] (issue: {es-issue}99297[#99297]) + +ES|QL:: +* ESQL: Change "substring" function to not return null on empty string {es-pull}109174[#109174] +* ESQL: Fix Join references {es-pull}109989[#109989] +* ESQL: Fix LOOKUP attribute shadowing {es-pull}109807[#109807] (issue: {es-issue}109392[#109392]) +* ESQL: Fix Max doubles bug with negatives and add tests for Max and Min {es-pull}110586[#110586] +* ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs` {es-pull}109205[#109205] (issue: {es-issue}109198[#109198]) +* ESQL: Fix equals `hashCode` for functions {es-pull}107947[#107947] (issue: {es-issue}104393[#104393]) +* ESQL: Fix variable shadowing when pushing down past Project {es-pull}108360[#108360] (issue: {es-issue}108008[#108008]) +* ESQL: Validate unique plan attribute names {es-pull}110488[#110488] (issue: {es-issue}110541[#110541]) +* ESQL: change from quoting from backtick to quote {es-pull}108395[#108395] +* ESQL: make named params objects truly per request {es-pull}110046[#110046] (issue: {es-issue}110028[#110028]) +* ES|QL: Fix DISSECT that overwrites input {es-pull}110201[#110201] (issue: {es-issue}110184[#110184]) +* ES|QL: limit query depth to 500 levels {es-pull}108089[#108089] (issue: {es-issue}107752[#107752]) +* ES|QL: reduce max expression depth to 400 {es-pull}111186[#111186] (issue: {es-issue}109846[#109846]) +* Fix ST_DISTANCE Lucene push-down for complex predicates {es-pull}110391[#110391] (issue: {es-issue}110349[#110349]) +* Fix `ClassCastException` with MV_EXPAND on missing field {es-pull}110096[#110096] (issue: {es-issue}109974[#109974]) +* Fix bug in union-types with type-casting in grouping key of STATS {es-pull}110476[#110476] (issues: {es-issue}109922[#109922], {es-issue}110477[#110477]) +* Fix for union-types for multiple columns with the same name {es-pull}110793[#110793] (issues: {es-issue}110490[#110490], {es-issue}109916[#109916]) +* [ESQL] Count_distinct(_source) should return a 400 {es-pull}110824[#110824] +* [ESQL] Fix parsing of large magnitude negative numbers {es-pull}110665[#110665] (issue: {es-issue}104323[#104323]) +* [ESQL] Migrate `SimplifyComparisonArithmetics` optimization {es-pull}109256[#109256] (issues: {es-issue}108388[#108388], {es-issue}108743[#108743]) + +Engine:: +* Async close of `IndexShard` {es-pull}108145[#108145] + +Highlighting:: +* Fix issue with returning incomplete fragment for plain highlighter {es-pull}110707[#110707] + +ILM+SLM:: +* Allow `read_slm` to call GET /_slm/status {es-pull}108333[#108333] + +Indices APIs:: +* Create a new `NodeRequest` for every `NodesDataTiersUsageTransport` use {es-pull}108379[#108379] + +Infra/Core:: +* Add a cluster listener to fix missing node features after upgrading from a version prior to 8.13 {es-pull}110710[#110710] (issue: {es-issue}109254[#109254]) +* Add bounds checking to parsing ISO8601 timezone offset values {es-pull}108672[#108672] +* Fix native preallocate to actually run {es-pull}110851[#110851] +* Ignore additional cpu.stat fields {es-pull}108019[#108019] (issue: {es-issue}107983[#107983]) +* Specify parse index when error occurs on multiple datetime parses {es-pull}108607[#108607] + +Infra/Metrics:: +* Provide document size reporter with `MapperService` {es-pull}109794[#109794] + +Infra/Node Lifecycle:: +* Expose `?master_timeout` on get-shutdown API {es-pull}108886[#108886] +* Fix serialization of put-shutdown request {es-pull}107862[#107862] (issue: {es-issue}107857[#107857]) +* Support wait indefinitely for search tasks to complete on node shutdown {es-pull}107426[#107426] + +Infra/REST API:: +* Add some missing timeout params to REST API specs {es-pull}108761[#108761] +* Consider `error_trace` supported by all endpoints {es-pull}109613[#109613] (issue: {es-issue}109612[#109612]) + +Ingest Node:: +* Fix Dissect with leading non-ascii characters {es-pull}111184[#111184] +* Fix enrich policy runner exception handling on empty segments response {es-pull}111290[#111290] +* GeoIP tasks should wait longer for master {es-pull}108410[#108410] +* Removing the use of Stream::peek from `GeoIpDownloader::cleanDatabases` {es-pull}110666[#110666] +* Simulate should succeed if `ignore_missing_pipeline` {es-pull}108106[#108106] (issue: {es-issue}107314[#107314]) + +Machine Learning:: +* Allow deletion of the ELSER inference service when reference in ingest {es-pull}108146[#108146] +* Avoid `InferenceRunner` deadlock {es-pull}109551[#109551] +* Correctly handle duplicate model ids for the `_cat` trained models api and usage statistics {es-pull}109126[#109126] +* Do not use global ordinals strategy if the leaf reader context cannot be obtained {es-pull}108459[#108459] +* Fix NPE in trained model assignment updater {es-pull}108942[#108942] +* Fix serialising inference delete response {es-pull}109384[#109384] +* Fix "stack use after scope" memory error {ml-pull}2673[#2673] +* Fix trailing slash in `ml.get_categories` specification {es-pull}110146[#110146] +* Handle any exception thrown by inference {ml-pull}2680[#2680] +* Increase response size limit for batched requests {es-pull}110112[#110112] +* Offload request to generic threadpool {es-pull}109104[#109104] (issue: {es-issue}109100[#109100]) +* Propagate accurate deployment timeout {es-pull}109534[#109534] (issue: {es-issue}109407[#109407]) +* Refactor TextEmbeddingResults to use primitives rather than objects {es-pull}108161[#108161] +* Require question to be non-null in `QuestionAnsweringConfig` {es-pull}107972[#107972] +* Start Trained Model Deployment API request query params now override body params {es-pull}109487[#109487] +* Suppress deprecation warnings from ingest pipelines when deleting trained model {es-pull}108679[#108679] (issue: {es-issue}105004[#105004]) +* Use default translog durability on AD results index {es-pull}108999[#108999] +* Use the multi node routing action for internal inference services {es-pull}109358[#109358] +* [Inference API] Extract optional long instead of integer in `RateLimitSettings#of` {es-pull}108602[#108602] +* [Inference API] Fix serialization for inference delete endpoint response {es-pull}110431[#110431] +* [Inference API] Replace `model_id` with `inference_id` in inference API except when stored {es-pull}111366[#111366] + +Mapping:: +* Fix off by one error when handling null values in range fields {es-pull}107977[#107977] (issue: {es-issue}107282[#107282]) +* Limit number of synonym rules that can be created {es-pull}109981[#109981] (issue: {es-issue}108785[#108785]) +* Propagate mapper builder context flags across nested mapper builder context creation {es-pull}109963[#109963] +* `DenseVectorFieldMapper` fixed typo {es-pull}108065[#108065] + +Network:: +* Use proper executor for failing requests when connection closes {es-pull}109236[#109236] (issue: {es-issue}109225[#109225]) +* `NoSuchRemoteClusterException` should not be thrown when a remote is configured {es-pull}107435[#107435] (issue: {es-issue}107381[#107381]) + +Packaging:: +* Adding override for lintian false positive on `libvec.so` {es-pull}108521[#108521] (issue: {es-issue}108514[#108514]) + +Ranking:: +* Fix score count validation in reranker response {es-pull}111424[#111424] (issue: {es-issue}111202[#111202]) + +Rollup:: +* Fix trailing slash in two rollup specifications {es-pull}110176[#110176] + +Search:: +* Adding score from `RankDoc` to `SearchHit` {es-pull}108870[#108870] +* Better handling of multiple rescorers clauses with LTR {es-pull}109071[#109071] +* Correct query profiling for conjunctions {es-pull}108122[#108122] (issue: {es-issue}108116[#108116]) +* Fix `DecayFunctions'` `toString` {es-pull}107415[#107415] (issue: {es-issue}100870[#100870]) +* Fix leak in collapsing search results {es-pull}110927[#110927] +* Fork freeing search/scroll contexts to GENERIC pool {es-pull}109481[#109481] + +Security:: +* Add permission to secure access to certain config files {es-pull}107827[#107827] +* Add permission to secure access to certain config files specified by settings {es-pull}108895[#108895] +* Fix trappy timeouts in security settings APIs {es-pull}109233[#109233] + +Snapshot/Restore:: +* Stricter failure handling in multi-repo get-snapshots request handling {es-pull}107191[#107191] + +TSDB:: +* Sort time series indices by time range in `GetDataStreams` API {es-pull}107967[#107967] (issue: {es-issue}102088[#102088]) + +Transform:: +* Always pick the user `maxPageSize` value {es-pull}109876[#109876] (issue: {es-issue}109844[#109844]) +* Exit gracefully when deleted {es-pull}107917[#107917] (issue: {es-issue}107266[#107266]) +* Fix NPE during destination index creation {es-pull}108891[#108891] (issue: {es-issue}108890[#108890]) +* Forward `indexServiceSafe` exception to listener {es-pull}108517[#108517] (issue: {es-issue}108418[#108418]) +* Halt Indexer on Stop/Abort API {es-pull}107792[#107792] +* Handle `IndexNotFoundException` {es-pull}108394[#108394] (issue: {es-issue}107263[#107263]) +* Prevent concurrent jobs during cleanup {es-pull}109047[#109047] +* Redirect `VersionConflict` to reset code {es-pull}108070[#108070] +* Reset max page size to settings value {es-pull}109449[#109449] (issue: {es-issue}109308[#109308]) + +Vector Search:: +* Ensure vector similarity correctly limits `inner_hits` returned for nested kNN {es-pull}111363[#111363] (issue: {es-issue}111093[#111093]) +* Ensure we return non-negative scores when scoring scalar dot-products {es-pull}108522[#108522] + +Watcher:: +* Avoiding running watch jobs in TickerScheduleTriggerEngine if it is paused {es-pull}110061[#110061] (issue: {es-issue}105933[#105933]) + +[[deprecation-8.15.0]] +[float] +=== Deprecations + +ILM+SLM:: +* Deprecate using slm privileges to access ilm {es-pull}110540[#110540] + +Infra/Settings:: +* `ParseHeapRatioOrDeprecatedByteSizeValue` for `indices.breaker.total.limit` {es-pull}110236[#110236] + +Machine Learning:: +* Deprecate `text_expansion` and `weighted_tokens` queries {es-pull}109880[#109880] + +[[enhancement-8.15.0]] +[float] +=== Enhancements + +Aggregations:: +* Aggs: Scripted metric allow list {es-pull}109444[#109444] +* Enable inter-segment concurrency for low cardinality numeric terms aggs {es-pull}108306[#108306] +* Increase size of big arrays only when there is an actual value in the aggregators {es-pull}107764[#107764] +* Increase size of big arrays only when there is an actual value in the aggregators (Analytics module) {es-pull}107813[#107813] +* Optimise `BinaryRangeAggregator` for single value fields {es-pull}108016[#108016] +* Optimise cardinality aggregations for single value fields {es-pull}107892[#107892] +* Optimise composite aggregations for single value fields {es-pull}107897[#107897] +* Optimise few metric aggregations for single value fields {es-pull}107832[#107832] +* Optimise histogram aggregations for single value fields {es-pull}107893[#107893] +* Optimise multiterms aggregation for single value fields {es-pull}107937[#107937] +* Optimise terms aggregations for single value fields {es-pull}107930[#107930] +* Speed up collecting zero document string terms {es-pull}110922[#110922] + +Allocation:: +* Log shard movements {es-pull}105829[#105829] +* Support effective watermark thresholds in node stats API {es-pull}107244[#107244] (issue: {es-issue}106676[#106676]) + +Application:: +* Add Create or update query rule API call {es-pull}109042[#109042] +* Rename rule query and add support for multiple rulesets {es-pull}108831[#108831] +* Support multiple associated groups for TopN {es-pull}108409[#108409] (issue: {es-issue}108018[#108018]) +* [Connector API] Change `UpdateConnectorFiltering` API to have better defaults {es-pull}108612[#108612] + +Authentication:: +* Expose API Key cache metrics {es-pull}109078[#109078] + +Authorization:: +* Cluster state role mapper file settings service {es-pull}107886[#107886] +* Cluster-state based Security role mapper {es-pull}107410[#107410] +* Introduce role description field {es-pull}107088[#107088] +* [Osquery] Extend `kibana_system` role with an access to new `osquery_manager` index {es-pull}108849[#108849] + +Data streams:: +* Add metrics@custom component template to metrics-*-* index template {es-pull}109540[#109540] (issue: {es-issue}109475[#109475]) +* Apm-data: enable plugin by default {es-pull}108860[#108860] +* Apm-data: ignore malformed fields, and too many dynamic fields {es-pull}108444[#108444] +* Apm-data: improve default pipeline performance {es-pull}108396[#108396] (issue: {es-issue}108290[#108290]) +* Apm-data: improve indexing resilience {es-pull}108227[#108227] +* Apm-data: increase priority above Fleet templates {es-pull}108885[#108885] +* Apm-data: increase version for templates {es-pull}108340[#108340] +* Apm-data: set codec: best_compression for logs-apm.* data streams {es-pull}108862[#108862] +* Remove `default_field: message` from metrics index templates {es-pull}110651[#110651] + +Distributed:: +* Add `wait_for_completion` parameter to delete snapshot request {es-pull}109462[#109462] (issue: {es-issue}101300[#101300]) +* Improve mechanism for extracting the result of a `PlainActionFuture` {es-pull}110019[#110019] (issue: {es-issue}108125[#108125]) + +ES|QL:: +* Add `BlockHash` for 3 `BytesRefs` {es-pull}108165[#108165] +* Allow `LuceneSourceOperator` to early terminate {es-pull}108820[#108820] +* Check if `CsvTests` required capabilities exist {es-pull}108684[#108684] +* ESQL: Add aggregates node level reduction {es-pull}107876[#107876] +* ESQL: Add more time span units {es-pull}108300[#108300] +* ESQL: Implement LOOKUP, an "inline" enrich {es-pull}107987[#107987] (issue: {es-issue}107306[#107306]) +* ESQL: Renamed `TopList` to Top {es-pull}110347[#110347] +* ESQL: Union Types Support {es-pull}107545[#107545] (issue: {es-issue}100603[#100603]) +* ESQL: add REPEAT string function {es-pull}109220[#109220] +* ES|QL Add primitive float support to the Compute Engine {es-pull}109746[#109746] (issue: {es-issue}109178[#109178]) +* ES|QL Add primitive float variants of all aggregators to the compute engine {es-pull}109781[#109781] +* ES|QL: vectorize eval {es-pull}109332[#109332] +* Optimize ST_DISTANCE filtering with Lucene circle intersection query {es-pull}110102[#110102] (issue: {es-issue}109972[#109972]) +* Optimize for single value in ordinals grouping {es-pull}108118[#108118] +* Rewrite away type converting functions that do not convert types {es-pull}108713[#108713] (issue: {es-issue}107716[#107716]) +* ST_DISTANCE Function {es-pull}108764[#108764] (issue: {es-issue}108212[#108212]) +* Support metrics counter types in ESQL {es-pull}107877[#107877] +* [ESQL] CBRT function {es-pull}108574[#108574] +* [ES|QL] Convert string to datetime when the other size of an arithmetic operator is `date_period` or `time_duration` {es-pull}108455[#108455] +* [ES|QL] Support Named and Positional Parameters in `EsqlQueryRequest` {es-pull}108421[#108421] (issue: {es-issue}107029[#107029]) +* [ES|QL] `weighted_avg` {es-pull}109993[#109993] + +Engine:: +* Drop shards close timeout when stopping node. {es-pull}107978[#107978] (issue: {es-issue}107938[#107938]) +* Update translog `writeLocation` for `flushListener` after commit {es-pull}109603[#109603] + +Geo:: +* Optimize `GeoBounds` and `GeoCentroid` aggregations for single value fields {es-pull}107663[#107663] + +Health:: +* Log details of non-green indicators in `HealthPeriodicLogger` {es-pull}108266[#108266] + +Highlighting:: +* Unified Highlighter to support matched_fields {es-pull}107640[#107640] (issue: {es-issue}5172[#5172]) + +Infra/Core:: +* Add allocation explain output for THROTTLING shards {es-pull}109563[#109563] +* Create custom parser for ISO-8601 datetimes {es-pull}106486[#106486] (issue: {es-issue}102063[#102063]) +* Extend ISO8601 datetime parser to specify forbidden fields, allowing it to be used on more formats {es-pull}108606[#108606] +* add Elastic-internal stable bridge api for use by Logstash {es-pull}108171[#108171] + +Infra/Metrics:: +* Add auto-sharding APM metrics {es-pull}107593[#107593] +* Add request metric to `RestController` to track success/failure (by status code) {es-pull}109957[#109957] +* Allow RA metrics to be reported upon parsing completed or accumulated {es-pull}108726[#108726] +* Provide the `DocumentSizeReporter` with index mode {es-pull}108947[#108947] +* Return noop instance `DocSizeObserver` for updates with scripts {es-pull}108856[#108856] + +Ingest Node:: +* Add `continent_code` support to the geoip processor {es-pull}108780[#108780] (issue: {es-issue}85820[#85820]) +* Add support for the 'Connection Type' database to the geoip processor {es-pull}108683[#108683] +* Add support for the 'Domain' database to the geoip processor {es-pull}108639[#108639] +* Add support for the 'ISP' database to the geoip processor {es-pull}108651[#108651] +* Adding `hits_time_in_millis` and `misses_time_in_millis` to enrich cache stats {es-pull}107579[#107579] +* Adding `user_type` support for the enterprise database for the geoip processor {es-pull}108687[#108687] +* Adding human readable times to geoip stats {es-pull}107647[#107647] +* Include doc size info in ingest stats {es-pull}107240[#107240] (issue: {es-issue}106386[#106386]) +* Make ingest byte stat names more descriptive {es-pull}108786[#108786] +* Return ingest byte stats even when 0-valued {es-pull}108796[#108796] +* Test pipeline run after reroute {es-pull}108693[#108693] + +Logs:: +* Introduce a node setting controlling the activation of the `logs` index mode in logs@settings component template {es-pull}109025[#109025] (issue: {es-issue}108762[#108762]) +* Support index sorting with nested fields {es-pull}110251[#110251] (issue: {es-issue}107349[#107349]) + +Machine Learning:: +* Add Anthropic messages integration to Inference API {es-pull}109893[#109893] +* Add `sparse_vector` query {es-pull}108254[#108254] +* Add model download progress to the download task status {es-pull}107676[#107676] +* Add rate limiting support for the Inference API {es-pull}107706[#107706] +* Add the rerank task to the Elasticsearch internal inference service {es-pull}108452[#108452] +* Default the HF service to cosine similarity {es-pull}109967[#109967] +* GA the update trained model action {es-pull}108868[#108868] +* Handle the "JSON memory allocator bytes" field {es-pull}109653[#109653] +* Inference Processor: skip inference when all fields are missing {es-pull}108131[#108131] +* Log 'No statistics at.. ' message as a warning {ml-pull}2684[#2684] +* Optimise frequent item sets aggregation for single value fields {es-pull}108130[#108130] +* Sentence Chunker {es-pull}110334[#110334] +* [Inference API] Add Amazon Bedrock Support to Inference API {es-pull}110248[#110248] +* [Inference API] Add Mistral Embeddings Support to Inference API {es-pull}109194[#109194] +* [Inference API] Check for related pipelines on delete inference endpoint {es-pull}109123[#109123] + +Mapping:: +* Add ignored field values to synthetic source {es-pull}107567[#107567] +* Apply FLS to the contents of `IgnoredSourceFieldMapper` {es-pull}109931[#109931] +* Binary field enables doc values by default for index mode with synthe… {es-pull}107739[#107739] (issue: {es-issue}107554[#107554]) +* Feature/annotated text store defaults {es-pull}107922[#107922] (issue: {es-issue}107734[#107734]) +* Handle `ignore_above` in synthetic source for flattened fields {es-pull}110214[#110214] +* Opt in keyword field into fallback synthetic source if needed {es-pull}110016[#110016] +* Opt in number fields into fallback synthetic source when doc values a… {es-pull}110160[#110160] +* Reflect latest changes in synthetic source documentation {es-pull}109501[#109501] +* Store source for fields in objects with `dynamic` override {es-pull}108911[#108911] +* Store source for nested objects {es-pull}108818[#108818] +* Support synthetic source for `geo_point` when `ignore_malformed` is used {es-pull}109651[#109651] +* Support synthetic source for `scaled_float` and `unsigned_long` when `ignore_malformed` is used {es-pull}109506[#109506] +* Support synthetic source for date fields when `ignore_malformed` is used {es-pull}109410[#109410] +* Support synthetic source together with `ignore_malformed` in histogram fields {es-pull}109882[#109882] +* Track source for arrays of objects {es-pull}108417[#108417] (issue: {es-issue}90708[#90708]) +* Track synthetic source for disabled objects {es-pull}108051[#108051] + +Network:: +* Detect long-running tasks on network threads {es-pull}109204[#109204] + +Ranking:: +* Enabling profiling for `RankBuilders` and adding tests for RRF {es-pull}109470[#109470] + +Relevance:: +* [Query Rules] Add API calls to get or delete individual query rules within a ruleset {es-pull}109554[#109554] +* [Query Rules] Require Enterprise License for Query Rules {es-pull}109634[#109634] + +Search:: +* Add AVX-512 optimised vector distance functions for int7 on x64 {es-pull}109084[#109084] +* Add `SparseVectorStats` {es-pull}108793[#108793] +* Add `_name` support for top level `knn` clauses {es-pull}107645[#107645] (issues: {es-issue}106254[#106254], {es-issue}107448[#107448]) +* Add a SIMD (AVX2) optimised vector distance function for int7 on x64 {es-pull}108088[#108088] +* Add min/max range of the `event.ingested` field to cluster state for searchable snapshots {es-pull}106252[#106252] +* Add per-field KNN vector format to Index Segments API {es-pull}107216[#107216] +* Add support for hiragana_uppercase & katakana_uppercase token filters in kuromoji analysis plugin {es-pull}106553[#106553] +* Adding support for explain in rrf {es-pull}108682[#108682] +* Allow rescorer with field collapsing {es-pull}107779[#107779] (issue: {es-issue}27243[#27243]) +* Cut over stored fields to ZSTD for compression {es-pull}103374[#103374] +* Limit the value in prefix query {es-pull}108537[#108537] (issue: {es-issue}108486[#108486]) +* Make dense vector field type updatable {es-pull}106591[#106591] +* Multivalue Sparse Vector Support {es-pull}109007[#109007] + +Security:: +* Add bulk delete roles API {es-pull}110383[#110383] +* Remote cluster - API key security model - cluster privileges {es-pull}107493[#107493] + +Snapshot/Restore:: +* Denser in-memory representation of `ShardBlobsToDelete` {es-pull}109848[#109848] +* Log repo UUID at generation/registration time {es-pull}109672[#109672] +* Make repository analysis API available to non-operators {es-pull}110179[#110179] (issue: {es-issue}100318[#100318]) +* Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics {es-pull}109657[#109657] + +Stats:: +* DocsStats: Add human readable bytesize {es-pull}109720[#109720] + +TSDB:: +* Optimise `time_series` aggregation for single value fields {es-pull}107990[#107990] +* Support `ignore_above` on keyword dimensions {es-pull}110337[#110337] + +Vector Search:: +* Adding hamming distance function to painless for `dense_vector` fields {es-pull}109359[#109359] +* Support k parameter for knn query {es-pull}110233[#110233] (issue: {es-issue}108473[#108473]) + +[[feature-8.15.0]] [float] -=== Known issues +=== New features + +Aggregations:: +* Opt `scripted_metric` out of parallelization {es-pull}109597[#109597] + +Application:: +* [Connector API] Add claim sync job endpoint {es-pull}109480[#109480] + +ES|QL:: +* ESQL: Add `ip_prefix` function {es-pull}109070[#109070] (issue: {es-issue}99064[#99064]) +* ESQL: Introduce a casting operator, `::` {es-pull}107409[#107409] +* ESQL: `top_list` aggregation {es-pull}109386[#109386] (issue: {es-issue}109213[#109213]) +* ESQL: add Arrow dataframes output format {es-pull}109873[#109873] +* Reapply "ESQL: Expose "_ignored" metadata field" {es-pull}108871[#108871] + +Infra/REST API:: +* Add a capabilities API to check node and cluster capabilities {es-pull}106820[#106820] + +Ingest Node:: +* Directly download commercial ip geolocation databases from providers {es-pull}110844[#110844] +* Mark the Redact processor as Generally Available {es-pull}110395[#110395] + +Logs:: +* Introduce logs index mode as Tech Preview {es-pull}108896[#108896] (issue: {es-issue}108896[#108896]) + +Machine Learning:: +* Add support for Azure AI Studio embeddings and completions to the inference service. {es-pull}108472[#108472] + +Mapping:: +* Add `semantic_text` field type and `semantic` query {es-pull}110338[#110338] +* Add generic fallback implementation for synthetic source {es-pull}108222[#108222] +* Add synthetic source support for `geo_shape` via fallback implementation {es-pull}108881[#108881] +* Add synthetic source support for binary fields {es-pull}107549[#107549] +* Enable fallback synthetic source by default {es-pull}109370[#109370] (issue: {es-issue}106460[#106460]) +* Enable fallback synthetic source for `point` and `shape` {es-pull}109312[#109312] +* Enable fallback synthetic source for `token_count` {es-pull}109044[#109044] +* Implement synthetic source support for annotated text field {es-pull}107735[#107735] +* Implement synthetic source support for range fields {es-pull}107081[#107081] +* Support arrays in fallback synthetic source implementation {es-pull}108878[#108878] +* Support synthetic source for `aggregate_metric_double` when ignore_malf… {es-pull}108746[#108746] + +Ranking:: +* Add text similarity reranker retriever {es-pull}109813[#109813] + +Relevance:: +* Mark Query Rules as GA {es-pull}110004[#110004] + +Search:: +* Add new int4 quantization to dense_vector {es-pull}109317[#109317] +* Adding RankFeature search phase implementation {es-pull}108538[#108538] +* Adding aggregations support for the `_ignored` field {es-pull}101373[#101373] (issue: {es-issue}59946[#59946]) +* Update Lucene version to 9.11 {es-pull}109219[#109219] + +Security:: +* Query Roles API {es-pull}108733[#108733] + +Transform:: +* Introduce _transform/_node_stats API {es-pull}107279[#107279] + +Vector Search:: +* Adds new `bit` `element_type` for `dense_vectors` {es-pull}110059[#110059] + +[[upgrade-8.15.0]] +[float] +=== Upgrades + +Infra/Plugins:: +* Update ASM to 9.7 for plugin scanner {es-pull}108822[#108822] (issue: {es-issue}108776[#108776]) + +Ingest Node:: +* Bump Tika dependency to 2.9.2 {es-pull}108144[#108144] + +Network:: +* Upgrade to Netty 4.1.109 {es-pull}108155[#108155] + +Search:: +* Upgrade to Lucene-9.11.1 {es-pull}110234[#110234] + +Security:: +* Upgrade bouncy castle (non-fips) to 1.78.1 {es-pull}108223[#108223] + +Snapshot/Restore:: +* Bump jackson version in modules:repository-azure {es-pull}109717[#109717] + -* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. -In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` -process to reclaim memory. This can cause inference requests to fail. -Elasticsearch will automatically restart the `pytorch_inference` process -after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 0ed01ff422700..007dd740f34cf 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -68,6 +68,31 @@ improves query and merge speed significantly when compared to raw vectors. {es-pull}109317[#109317] +[discrete] +[[esql_inlinestats]] +=== ESQL: INLINESTATS +This adds the `INLINESTATS` command to ESQL which performs a STATS and +then enriches the results into the output stream. So, this query: + +[source,esql] +---- +FROM test +| INLINESTATS m=MAX(a * b) BY b +| WHERE m == a * b +| SORT a DESC, b DESC +| LIMIT 3 +---- + +Produces output like: + +| a | b | m | +| --- | --- | ----- | +| 99 | 999 | 98901 | +| 99 | 998 | 98802 | +| 99 | 997 | 98703 | + +{es-pull}109583[#109583] + [discrete] [[mark_query_rules_as_ga]] === Mark Query Rules as GA @@ -112,6 +137,18 @@ The Redact processor uses the Grok rules engine to obscure text in the input doc {es-pull}110395[#110395] +[discrete] +[[always_allow_rebalancing_by_default]] +=== Always allow rebalancing by default +In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to +`indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was +appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has +better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some +shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From +version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled. + +{es-pull}111015[#111015] + // end::notable-highlights[] From b31feb300ff301190389767addc85f740763ad1d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 8 Aug 2024 10:35:34 -0700 Subject: [PATCH 11/37] Only emit product origin in deprecation log if present (#111683) The elastic product origin may not always be present when deprecation messages are emitted. This commit changes the log message created for deprecations to only emit the product origin field if it is not empty. closes #81757 --- docs/changelog/111683.yaml | 6 ++++++ .../common/logging/DeprecatedMessage.java | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/111683.yaml diff --git a/docs/changelog/111683.yaml b/docs/changelog/111683.yaml new file mode 100644 index 0000000000000..cbb2e5ad71ddc --- /dev/null +++ b/docs/changelog/111683.yaml @@ -0,0 +1,6 @@ +pr: 111683 +summary: Only emit product origin in deprecation log if present +area: Infra/Logging +type: bug +issues: + - 81757 diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java index 80aeaffaabec5..7f94e8d3d1824 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java @@ -62,10 +62,12 @@ private static ESLogMessage getEsLogMessage( .field(KEY_FIELD_NAME, key) .field("elasticsearch.event.category", category.name().toLowerCase(Locale.ROOT)); - if (Strings.isNullOrEmpty(xOpaqueId)) { - return esLogMessage; + if (Strings.isNullOrEmpty(xOpaqueId) == false) { + esLogMessage.field(X_OPAQUE_ID_FIELD_NAME, xOpaqueId); } - - return esLogMessage.field(X_OPAQUE_ID_FIELD_NAME, xOpaqueId).field(ELASTIC_ORIGIN_FIELD_NAME, productOrigin); + if (Strings.isNullOrEmpty(productOrigin) == false) { + esLogMessage.field(ELASTIC_ORIGIN_FIELD_NAME, productOrigin); + } + return esLogMessage; } } From 801c8640efa04546ff1e7877bf7486e2f728d2d4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 9 Aug 2024 04:31:13 +1000 Subject: [PATCH 12/37] Remove 8.14 from branches.json --- branches.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/branches.json b/branches.json index b852cd1fa5dbd..1d860501cbc87 100644 --- a/branches.json +++ b/branches.json @@ -7,9 +7,6 @@ { "branch": "8.15" }, - { - "branch": "8.14" - }, { "branch": "7.17" } From ea3379e8a7434156198ccf17ecdb597cfc872c4f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 9 Aug 2024 06:28:27 +1000 Subject: [PATCH 13/37] Mute org.elasticsearch.tdigest.ComparisonTests testSparseGaussianDistribution #111721 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 018685a194db7..e439759d75dab 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -122,6 +122,9 @@ tests: - class: org.elasticsearch.xpack.inference.integration.ModelRegistryIT method: testGetModel issue: https://github.com/elastic/elasticsearch/issues/111570 +- class: org.elasticsearch.tdigest.ComparisonTests + method: testSparseGaussianDistribution + issue: https://github.com/elastic/elasticsearch/issues/111721 # Examples: # From 1163d2e4f957126b7847fd6ee7e6544f860da128 Mon Sep 17 00:00:00 2001 From: Mikhail Berezovskiy Date: Thu, 8 Aug 2024 13:32:20 -0700 Subject: [PATCH 14/37] Rename streamContent/Separator to bulkContent/Separator (#111716) Rename `xContent.streamSeparator()` and `RestHandler.supportsStreamContent()` to `xContent.bulkSeparator()` and `RestHandler.supportsBulkContent()`. I want to reserve use of "supportsStreamContent" for current work in HTTP layer to [support incremental content handling](https://github.com/elastic/elasticsearch/pull/111438) besides fully aggregated byte buffers. `supportsStreamContent` would indicate that handler can parse chunks of http content as they arrive. --- .../xcontent/provider/cbor/CborXContentImpl.java | 4 ++-- .../xcontent/provider/json/JsonXContentImpl.java | 2 +- .../xcontent/provider/smile/SmileXContentImpl.java | 2 +- .../xcontent/provider/yaml/YamlXContentImpl.java | 4 ++-- .../java/org/elasticsearch/xcontent/XContent.java | 2 +- .../script/mustache/MultiSearchTemplateRequest.java | 4 ++-- .../mustache/RestMultiSearchTemplateAction.java | 2 +- .../elasticsearch/action/bulk/BulkRequestParser.java | 2 +- .../action/search/MultiSearchRequest.java | 6 +++--- .../org/elasticsearch/rest/FilterRestHandler.java | 4 ++-- .../java/org/elasticsearch/rest/RestController.java | 2 +- .../java/org/elasticsearch/rest/RestHandler.java | 8 ++++---- .../rest/action/document/RestBulkAction.java | 2 +- .../rest/action/search/RestMultiSearchAction.java | 2 +- .../elasticsearch/action/bulk/BulkRequestTests.java | 8 ++++---- .../rest/DeprecationRestHandlerTests.java | 12 ++++++------ .../org/elasticsearch/rest/RestControllerTests.java | 10 +++++----- .../rest/yaml/ClientYamlTestExecutionContext.java | 2 +- .../autodetect/writer/JsonDataToProcessWriter.java | 2 +- .../normalizer/output/NormalizerResultHandler.java | 2 +- .../ml/process/logging/CppLogMessageHandler.java | 2 +- .../xpack/ml/rest/job/RestPostDataAction.java | 2 +- .../writer/JsonDataToProcessWriterTests.java | 4 ++-- .../monitoring/exporter/http/HttpExportBulk.java | 4 ++-- .../rest/action/RestMonitoringBulkAction.java | 2 +- .../action/MonitoringBulkRequestTests.java | 10 +++++----- .../rest/action/RestMonitoringBulkActionTests.java | 4 ++-- .../RestMonitoringMigrateAlertsActionTests.java | 2 +- .../xpack/runtimefields/test/CoreTestTranslater.java | 2 +- 29 files changed, 57 insertions(+), 57 deletions(-) diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java index 2a8e7a4dfa12c..3aa8323eb5495 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java @@ -63,8 +63,8 @@ public XContentType type() { } @Override - public byte streamSeparator() { - throw new XContentParseException("cbor does not support stream parsing..."); + public byte bulkSeparator() { + throw new XContentParseException("cbor does not support bulk parsing..."); } @Override diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java index 2e4925b4a853e..ae494796c88cb 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java @@ -65,7 +65,7 @@ public XContentType type() { } @Override - public byte streamSeparator() { + public byte bulkSeparator() { return '\n'; } diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java index 3c774c582c638..83528980c2b52 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java @@ -65,7 +65,7 @@ public XContentType type() { } @Override - public byte streamSeparator() { + public byte bulkSeparator() { return (byte) 0xFF; } diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java index 6a22508ba51c6..6e1496bfffd7b 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java @@ -61,8 +61,8 @@ public XContentType type() { } @Override - public byte streamSeparator() { - throw new UnsupportedOperationException("yaml does not support stream parsing..."); + public byte bulkSeparator() { + throw new UnsupportedOperationException("yaml does not support bulk parsing..."); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java index 146f90b8e2510..56eb308eaebae 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java @@ -24,7 +24,7 @@ public interface XContent { */ XContentType type(); - byte streamSeparator(); + byte bulkSeparator(); @Deprecated boolean detectContent(byte[] bytes, int offset, int length); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index b97d1b00573f4..2e50d14b24d72 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -142,12 +142,12 @@ public static byte[] writeMultiLineFormat(MultiSearchTemplateRequest multiSearch MultiSearchRequest.writeSearchRequestParams(searchRequest, xContentBuilder); BytesReference.bytes(xContentBuilder).writeTo(output); } - output.write(xContent.streamSeparator()); + output.write(xContent.bulkSeparator()); try (XContentBuilder xContentBuilder = XContentBuilder.builder(xContent)) { templateRequest.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); BytesReference.bytes(xContentBuilder).writeTo(output); } - output.write(xContent.streamSeparator()); + output.write(xContent.bulkSeparator()); } return output.toByteArray(); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index d9466536ac46c..f7f68324907ff 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -94,7 +94,7 @@ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, b } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java index 75ab08de942dc..898bfd0e1652c 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java @@ -139,7 +139,7 @@ public void parse( XContent xContent = xContentType.xContent(); int line = 0; int from = 0; - byte marker = xContent.streamSeparator(); + byte marker = xContent.bulkSeparator(); // Bulk requests can contain a lot of repeated strings for the index, pipeline and routing parameters. This map is used to // deduplicate duplicate strings parsed for these parameters. While it does not prevent instantiating the duplicate strings, it // reduces their lifetime to the lifetime of this parse call instead of the lifetime of the full bulk request. diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index 4bf8206861a4d..fa22cb36b70ab 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -206,7 +206,7 @@ public static void readMultiLineFormat( TriFunction extraParamParser ) throws IOException { int from = 0; - byte marker = xContent.streamSeparator(); + byte marker = xContent.bulkSeparator(); while (true) { int nextMarker = findNextMarker(marker, from, data); if (nextMarker == -1) { @@ -343,7 +343,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest, writeSearchRequestParams(request, xContentBuilder); BytesReference.bytes(xContentBuilder).writeTo(output); } - output.write(xContent.streamSeparator()); + output.write(xContent.bulkSeparator()); try (XContentBuilder xContentBuilder = XContentBuilder.builder(xContent)) { if (request.source() != null) { request.source().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); @@ -353,7 +353,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest, } BytesReference.bytes(xContentBuilder).writeTo(output); } - output.write(xContent.streamSeparator()); + output.write(xContent.bulkSeparator()); } return output.toByteArray(); } diff --git a/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java b/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java index a3b3c9d7b17f1..e07d8aad04609 100644 --- a/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java @@ -48,8 +48,8 @@ public boolean allowsUnsafeBuffers() { } @Override - public boolean supportsContentStream() { - return delegate.supportsContentStream(); + public boolean supportsBulkContent() { + return delegate.supportsBulkContent(); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 3f9c0dbaa11d6..8592888d2dd03 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -432,7 +432,7 @@ private void dispatchRequest( } final XContentType xContentType = request.getXContentType(); // TODO consider refactoring to handler.supportsContentStream(xContentType). It is only used with JSON and SMILE - if (handler.supportsContentStream() + if (handler.supportsBulkContent() && XContentType.JSON != xContentType.canonical() && XContentType.SMILE != xContentType.canonical()) { channel.sendResponse( diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index 11208a24ceb10..474bf02b4b92b 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -39,11 +39,11 @@ default boolean canTripCircuitBreaker() { } /** - * Indicates if the RestHandler supports content as a stream. A stream would be multiple objects delineated by - * {@link XContent#streamSeparator()}. If a handler returns true this will affect the types of content that can be sent to - * this endpoint. + * Indicates if the RestHandler supports bulk content. A bulk request contains multiple objects + * delineated by {@link XContent#bulkSeparator()}. If a handler returns true this will affect + * the types of content that can be sent to this endpoint. */ - default boolean supportsContentStream() { + default boolean supportsBulkContent() { return false; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index 0bb97b1f51ff5..d213d4410c07c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -101,7 +101,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 0a7a4a9701a90..d45cbaecb5a62 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -209,7 +209,7 @@ public static void parseMultiLineRequest( } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index 5732e7f29f65d..61f5105301c30 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -290,13 +290,13 @@ public void testSmileIsSupported() throws IOException { builder.endObject(); builder.endObject(); } - out.write(xContentType.xContent().streamSeparator()); + out.write(xContentType.xContent().bulkSeparator()); try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { builder.startObject(); builder.field("field", "value"); builder.endObject(); } - out.write(xContentType.xContent().streamSeparator()); + out.write(xContentType.xContent().bulkSeparator()); data = out.bytes(); } @@ -327,7 +327,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException builder.endObject(); builder.endObject(); } - out.write(xContentType.xContent().streamSeparator()); + out.write(xContentType.xContent().bulkSeparator()); try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("doc").endObject(); @@ -338,7 +338,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException builder.field("upsert", values); builder.endObject(); } - out.write(xContentType.xContent().streamSeparator()); + out.write(xContentType.xContent().bulkSeparator()); data = out.bytes(); } BulkRequest bulkRequest = new BulkRequest(); diff --git a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java index 90d9f1bcac879..a1898082418e5 100644 --- a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java @@ -155,17 +155,17 @@ public void testInvalidHeaderValueEmpty() { expectThrows(IllegalArgumentException.class, () -> DeprecationRestHandler.requireValidHeader(blank)); } - public void testSupportsContentStreamTrue() { - when(handler.supportsContentStream()).thenReturn(true); + public void testSupportsBulkContentTrue() { + when(handler.supportsBulkContent()).thenReturn(true); assertTrue( - new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsContentStream() + new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent() ); } - public void testSupportsContentStreamFalse() { - when(handler.supportsContentStream()).thenReturn(false); + public void testSupportsBulkContentFalse() { + when(handler.supportsBulkContent()).thenReturn(false); assertFalse( - new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsContentStream() + new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent() ); } diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 67f42e6cf1808..391b2a08021fd 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -598,7 +598,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } }); @@ -637,7 +637,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } }); @@ -662,7 +662,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } }); @@ -688,7 +688,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } }); @@ -713,7 +713,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } @Override - public boolean supportsContentStream() { + public boolean supportsBulkContent() { return true; } }); diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index d13fd2f347f7b..1f4a4dcf9dbfa 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -200,7 +200,7 @@ private HttpEntity createEntity(List> bodies, Map indexRequests = new ArrayList<>(); new BulkRequestParser(false, RestApiVersion.current()).parse( From e23588046a888b3631955106ad334300dde7ceb5 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 8 Aug 2024 16:48:47 -0700 Subject: [PATCH 15/37] Pass allow security manager flag in gradle test policy setup plugin (#111725) The `java.security.manager=allow` system property is required when running tests on newer Java versions with the security manager deprecated. As such, it should be set in our `GradleTestPolicySetupPlugin` so that it's done for external plugin authors. --- .../elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java index a1da860abe26a..9593a281686e7 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java @@ -22,6 +22,9 @@ public void apply(Project project) { test.systemProperty("tests.gradle", true); test.systemProperty("tests.task", test.getPath()); + // Flag is required for later Java versions since our tests use a custom security manager + test.jvmArgs("-Djava.security.manager=allow"); + SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider(); // don't track these as inputs since they contain absolute paths and break cache relocatability nonInputProperties.systemProperty("gradle.dist.lib", gradle.getGradleHomeDir().getAbsolutePath() + "/lib"); From 585480fe44d453ffbd1d017bf592743430f06be3 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 9 Aug 2024 09:38:14 +0200 Subject: [PATCH 16/37] ESQL: Fix for overzealous validation in case of invalid mapped fields (#111475) Fix validation of fields mapped to different types in different indices and align with validation of fields of unsupported type. * Allow using multi-typed fields in KEEP and DROP, just like unsupported fields. * Explicitly invalidate using both these field kinds in RENAME. * Map both kinds of fields to UnsupportedAttribute to enforce consistency. * Consider convert functions containing valid multi-typed fields as resolved to avoid weird workarounds when resolving STATS. * Add a bunch of tests. --- docs/changelog/111475.yaml | 6 + docs/reference/esql/esql-multi-index.asciidoc | 8 +- .../xpack/esql/core/expression/Alias.java | 5 + .../esql/core/expression/NamedExpression.java | 4 + .../esql/core/expression/TypeResolutions.java | 68 ++++- .../core/expression/UnresolvedAttribute.java | 5 + .../esql/core/type/InvalidMappedField.java | 5 + .../src/main/resources/stats.csv-spec | 13 + .../src/main/resources/union_types.csv-spec | 251 ++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 6 + .../xpack/esql/analysis/Analyzer.java | 90 +++---- .../xpack/esql/analysis/Verifier.java | 17 +- .../function/UnresolvedFunction.java | 5 + .../convert/AbstractConvertFunction.java | 6 +- .../xpack/esql/plan/logical/Stats.java | 5 - .../xpack/esql/analysis/AnalyzerTests.java | 28 +- .../xpack/esql/analysis/VerifierTests.java | 163 ++++++++++++ .../test/esql/160_union_types.yml | 155 ++++++++++- .../rest-api-spec/test/esql/40_tsdb.yml | 16 +- 19 files changed, 738 insertions(+), 118 deletions(-) create mode 100644 docs/changelog/111475.yaml diff --git a/docs/changelog/111475.yaml b/docs/changelog/111475.yaml new file mode 100644 index 0000000000000..264c975444868 --- /dev/null +++ b/docs/changelog/111475.yaml @@ -0,0 +1,6 @@ +pr: 111475 +summary: "ESQL: Fix for overzealous validation in case of invalid mapped fields" +area: ES|QL +type: bug +issues: + - 111452 diff --git a/docs/reference/esql/esql-multi-index.asciidoc b/docs/reference/esql/esql-multi-index.asciidoc index cf98cbe959237..25874a132d93d 100644 --- a/docs/reference/esql/esql-multi-index.asciidoc +++ b/docs/reference/esql/esql-multi-index.asciidoc @@ -97,8 +97,7 @@ In addition, if the query refers to this unsupported field directly, the query f [source.merge.styled,esql] ---- FROM events_* -| KEEP @timestamp, client_ip, event_duration, message -| SORT @timestamp DESC +| SORT client_ip DESC ---- [source,bash] @@ -118,9 +117,8 @@ experimental::[] {esql} has a way to handle <>. When the same field is mapped to multiple types in multiple indices, the type of the field is understood to be a _union_ of the various types in the index mappings. As seen in the preceding examples, this _union type_ cannot be used in the results, -and cannot be referred to by the query --- except when it's passed to a type conversion function that accepts all the types in the _union_ and converts the field -to a single type. {esql} offers a suite of <> to achieve this. +and cannot be referred to by the query -- except in `KEEP`, `DROP` or when it's passed to a type conversion function that accepts all the types in +the _union_ and converts the field to a single type. {esql} offers a suite of <> to achieve this. In the above examples, the query can use a command like `EVAL client_ip = TO_IP(client_ip)` to resolve the union of `ip` and `keyword` to just `ip`. diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java index 2335a69d9ec88..e33f9b1c20527 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java @@ -109,6 +109,11 @@ public Nullability nullable() { return child.nullable(); } + @Override + protected TypeResolution resolveType() { + return child.resolveType(); + } + @Override public DataType dataType() { return child.dataType(); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java index e3e9a60180da7..266ad8e2bb051 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java @@ -55,6 +55,10 @@ public boolean synthetic() { return synthetic; } + /** + * Try to return either {@code this} if it is an {@link Attribute}, or a {@link ReferenceAttribute} to it otherwise. + * Return an {@link UnresolvedAttribute} if this is unresolved. + */ public abstract Attribute toAttribute(); @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index c3593e91c537e..ab05a71b0e1c6 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression.TypeResolution; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import java.util.Locale; import java.util.StringJoiner; @@ -176,19 +177,60 @@ public static TypeResolution isType( ParamOrdinal paramOrd, String... acceptedTypes ) { - return predicate.test(e.dataType()) || e.dataType() == NULL - ? TypeResolution.TYPE_RESOLVED - : new TypeResolution( - format( - null, - "{}argument of [{}] must be [{}], found value [{}] type [{}]", - paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ", - operationName, - acceptedTypesForErrorMsg(acceptedTypes), - name(e), - e.dataType().typeName() - ) - ); + return isType(e, predicate, operationName, paramOrd, false, acceptedTypes); + } + + public static TypeResolution isTypeOrUnionType( + Expression e, + Predicate predicate, + String operationName, + ParamOrdinal paramOrd, + String... acceptedTypes + ) { + return isType(e, predicate, operationName, paramOrd, true, acceptedTypes); + } + + public static TypeResolution isType( + Expression e, + Predicate predicate, + String operationName, + ParamOrdinal paramOrd, + boolean allowUnionTypes, + String... acceptedTypes + ) { + if (predicate.test(e.dataType()) || e.dataType() == NULL) { + return TypeResolution.TYPE_RESOLVED; + } + + // TODO: Shouldn't we perform widening of small numerical types here? + if (allowUnionTypes + && e instanceof FieldAttribute fa + && fa.field() instanceof InvalidMappedField imf + && imf.types().stream().allMatch(predicate)) { + return TypeResolution.TYPE_RESOLVED; + } + + return new TypeResolution( + errorStringIncompatibleTypes(operationName, paramOrd, name(e), e.dataType(), acceptedTypesForErrorMsg(acceptedTypes)) + ); + } + + private static String errorStringIncompatibleTypes( + String operationName, + ParamOrdinal paramOrd, + String argumentName, + DataType foundType, + String... acceptedTypes + ) { + return format( + null, + "{}argument of [{}] must be [{}], found value [{}] type [{}]", + paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ", + operationName, + acceptedTypesForErrorMsg(acceptedTypes), + argumentName, + foundType.typeName() + ); } private static String acceptedTypesForErrorMsg(String... acceptedTypes) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java index a6b519cc18525..d8a35adcbffde 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java @@ -77,6 +77,11 @@ public UnresolvedAttribute withUnresolvedMessage(String unresolvedMessage) { return new UnresolvedAttribute(source(), name(), id(), unresolvedMessage, resolutionMetadata()); } + @Override + protected TypeResolution resolveType() { + return new TypeResolution("unresolved attribute [" + name() + "]"); + } + @Override public DataType dataType() { throw new UnresolvedException("dataType", this); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java index 9b088cfb19f6c..9b3d7950c2a01 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java @@ -17,6 +17,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Representation of field mapped differently across indices. @@ -64,6 +65,10 @@ private InvalidMappedField(StreamInput in) throws IOException { this(in.readString(), in.readString(), in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class))); } + public Set types() { + return typesToIndices.keySet().stream().map(DataType::fromTypeName).collect(Collectors.toSet()); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(getName()); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 7b0dda9eb3669..eb373b6ddef6b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2018,6 +2018,19 @@ M null ; +shadowingInternalWithGroup2#[skip:-8.14.1,reason:implemented in 8.14] +FROM employees +| STATS x = MAX(emp_no), y = count(x) BY x = emp_no, x = gender +| SORT x ASC +; + +y:long | x:keyword + 33 | F + 57 | M + 0 | null +; + + shadowingTheGroup FROM employees | STATS gender = MAX(emp_no), gender = MIN(emp_no) BY gender diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index e1aa411702420..6d1d4c7892886 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1015,3 +1015,254 @@ client_ip:ip | event_duration:long | message:keyword | @timestamp:keywo ; # Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats + +multiIndexIndirectUseOfUnionTypesInSort +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword + null | 172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInEval +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| EVAL foo = event_duration > 1232381 +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:boolean + null | 172.21.0.5 | 1232382 | Disconnected | true +; + +multiIndexIndirectUseOfUnionTypesInRename +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: union_types_fix_rename_resolution +FROM sample_data, sample_data_ts_long +| RENAME message AS event_message +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | event_message:keyword + null | 172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInKeep +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| KEEP client_ip, event_duration, message +| SORT client_ip ASC +| LIMIT 1 +; + +client_ip:ip | event_duration:long | message:keyword +172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInWildcardKeep +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: union_types_fix_rename_resolution +FROM sample_data, sample_data_ts_long +| KEEP * +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword + null | 172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInWildcardKeep2 +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: union_types_fix_rename_resolution +FROM sample_data, sample_data_ts_long +| KEEP *e* +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword + null | 172.21.0.5 | 1232382 | Disconnected +; + + +multiIndexUseOfUnionTypesInKeep +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: union_types_fix_rename_resolution +FROM sample_data, sample_data_ts_long +| KEEP @timestamp +| LIMIT 1 +; + +@timestamp:null +null +; + +multiIndexUseOfUnionTypesInDrop +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: union_types_fix_rename_resolution +FROM sample_data, sample_data_ts_long +| DROP @timestamp +| SORT client_ip ASC +| LIMIT 1 +; + +client_ip:ip | event_duration:long | message:keyword +172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInWildcardDrop +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: union_types_fix_rename_resolution +FROM sample_data, sample_data_ts_long +| DROP *time* +| SORT client_ip ASC +| LIMIT 1 +; + +client_ip:ip | event_duration:long | message:keyword +172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInWhere +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| WHERE message == "Disconnected" +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword + null | 172.21.0.5 | 1232382 | Disconnected + null | 172.21.0.5 | 1232382 | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInDissect +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| DISSECT message "%{foo}" +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword + null | 172.21.0.5 | 1232382 | Disconnected | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInGrok +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| GROK message "%{WORD:foo}" +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword + null | 172.21.0.5 | 1232382 | Disconnected | Disconnected +; + +multiIndexIndirectUseOfUnionTypesInEnrich +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: enrich_load +FROM sample_data, sample_data_ts_long +| EVAL client_ip = client_ip::keyword +| ENRICH clientip_policy ON client_ip WITH env +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | event_duration:long | message:keyword | client_ip:keyword | env:keyword + null | 1232382 | Disconnected | 172.21.0.5 | Development +; + +multiIndexIndirectUseOfUnionTypesInStats +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| STATS foo = max(event_duration) BY client_ip +| SORT client_ip ASC +; + +foo:long | client_ip:ip + 1232382 | 172.21.0.5 + 2764889 | 172.21.2.113 + 3450233 | 172.21.2.162 + 8268153 | 172.21.3.15 +; + +multiIndexIndirectUseOfUnionTypesInInlineStats +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: inlinestats +FROM sample_data, sample_data_ts_long +| INLINESTATS foo = max(event_duration) +| SORT client_ip ASC +| LIMIT 1 +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:long + null | 172.21.0.5 | 1232382 | Disconnected | 8268153 +; + +multiIndexIndirectUseOfUnionTypesInLookup +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +required_capability: lookup_v4 +FROM sample_data, sample_data_ts_long +| SORT client_ip ASC +| LIMIT 1 +| EVAL int = (event_duration - 1232380)::integer +| LOOKUP int_number_names ON int +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword | int:integer | name:keyword + null | 172.21.0.5 | 1232382 | Disconnected | 2 | two +; + +multiIndexIndirectUseOfUnionTypesInMvExpand +// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution: +// make the csv tests work with multiple indices. +required_capability: union_types +FROM sample_data, sample_data_ts_long +| EVAL foo = MV_APPEND(message, message) +| SORT client_ip ASC +| LIMIT 1 +| MV_EXPAND foo +; + +@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword + null | 172.21.0.5 | 1232382 | Disconnected | Disconnected + null | 172.21.0.5 | 1232382 | Disconnected | Disconnected +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 42eebffc0375b..b1031f06a194d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -157,6 +157,12 @@ public enum Cap { */ UNION_TYPES_REMOVE_FIELDS, + /** + * Fix for union-types when renaming unrelated columns. + * https://github.com/elastic/elasticsearch/issues/111452 + */ + UNION_TYPES_FIX_RENAME_RESOLUTION, + /** * Fix a parsing issue where numbers below Long.MIN_VALUE threw an exception instead of parsing as doubles. * see Parsing large numbers is inconsistent #104323 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index d4c258ece0134..4a7120a1d3d92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -21,7 +21,6 @@ import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; -import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; @@ -62,7 +61,6 @@ import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.plan.TableIdentifier; -import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -439,6 +437,7 @@ private LogicalPlan resolveStats(Stats stats, List childrenOutput) { // e.g. STATS a ... GROUP BY a = x + 1 Holder changed = new Holder<>(false); List groupings = stats.groupings(); + List aggregates = stats.aggregates(); // first resolve groupings since the aggs might refer to them // trying to globally resolve unresolved attributes will lead to some being marked as unresolvable if (Resolvables.resolved(groupings) == false) { @@ -457,17 +456,17 @@ private LogicalPlan resolveStats(Stats stats, List childrenOutput) { } } - if (stats.expressionsResolved() == false) { - AttributeMap resolved = new AttributeMap<>(); + if (Resolvables.resolved(groupings) == false || (Resolvables.resolved(aggregates) == false)) { + ArrayList resolved = new ArrayList<>(); for (Expression e : groupings) { Attribute attr = Expressions.attribute(e); if (attr != null && attr.resolved()) { - resolved.put(attr, attr); + resolved.add(attr); } } - List resolvedList = NamedExpressions.mergeOutputAttributes(new ArrayList<>(resolved.keySet()), childrenOutput); - List newAggregates = new ArrayList<>(); + List resolvedList = NamedExpressions.mergeOutputAttributes(resolved, childrenOutput); + List newAggregates = new ArrayList<>(); for (NamedExpression aggregate : stats.aggregates()) { var agg = (NamedExpression) aggregate.transformUp(UnresolvedAttribute.class, ua -> { Expression ne = ua; @@ -802,9 +801,18 @@ private LogicalPlan resolveEnrich(Enrich enrich, List childrenOutput) String matchType = enrich.policy().getType(); DataType[] allowed = allowedEnrichTypes(matchType); if (Arrays.asList(allowed).contains(dataType) == false) { - String suffix = "only " + Arrays.toString(allowed) + " allowed for type [" + matchType + "]"; + String suffix = "only [" + + Arrays.stream(allowed).map(DataType::typeName).collect(Collectors.joining(", ")) + + "] allowed for type [" + + matchType + + "]"; resolved = ua.withUnresolvedMessage( - "Unsupported type [" + resolved.dataType() + "] for enrich matching field [" + ua.name() + "]; " + suffix + "Unsupported type [" + + resolved.dataType().typeName() + + "] for enrich matching field [" + + ua.name() + + "]; " + + suffix ); } } @@ -1057,24 +1065,19 @@ public static Expression castStringLiteral(Expression from, DataType target) { target, e.getMessage() ); - return new UnsupportedAttribute( - from.source(), - String.valueOf(from.fold()), - new UnsupportedEsField(String.valueOf(from.fold()), from.dataType().typeName()), - message - ); + return new UnresolvedAttribute(from.source(), String.valueOf(from.fold()), message); } } } /** * The EsqlIndexResolver will create InvalidMappedField instances for fields that are ambiguous (i.e. have multiple mappings). - * During ResolveRefs we do not convert these to UnresolvedAttribute instances, as we want to first determine if they can + * During {@link ResolveRefs} we do not convert these to UnresolvedAttribute instances, as we want to first determine if they can * instead be handled by conversion functions within the query. This rule looks for matching conversion functions and converts * those fields into MultiTypeEsField, which encapsulates the knowledge of how to convert these into a single type. * This knowledge will be used later in generating the FieldExtractExec with built-in type conversion. * Any fields which could not be resolved by conversion functions will be converted to UnresolvedAttribute instances in a later rule - * (See UnresolveUnionTypes below). + * (See {@link UnionTypesCleanup} below). */ private static class ResolveUnionTypes extends Rule { @@ -1094,7 +1097,7 @@ public LogicalPlan apply(LogicalPlan plan) { } }); - return plan.transformUp(LogicalPlan.class, p -> p.resolved() || p.childrenResolved() == false ? p : doRule(p)); + return plan.transformUp(LogicalPlan.class, p -> p.childrenResolved() == false ? p : doRule(p)); } private LogicalPlan doRule(LogicalPlan plan) { @@ -1110,24 +1113,6 @@ private LogicalPlan doRule(LogicalPlan plan) { return plan; } - // In ResolveRefs the aggregates are resolved from the groupings, which might have an unresolved MultiTypeEsField. - // Now that we have resolved those, we need to re-resolve the aggregates. - if (plan instanceof Aggregate agg) { - // TODO once inlinestats supports expressions in groups we'll likely need the same sort of extraction here - // If the union-types resolution occurred in a child of the aggregate, we need to check the groupings - plan = agg.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved); - - // Aggregates where the grouping key comes from a union-type field need to be resolved against the grouping key - Map resolved = new HashMap<>(); - for (Expression e : agg.groupings()) { - Attribute attr = Expressions.attribute(e); - if (attr != null && attr.resolved()) { - resolved.put(attr, e); - } - } - plan = plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> resolveAttribute(ua, resolved)); - } - // And add generated fields to EsRelation, so these new attributes will appear in the OutputExec of the Fragment // and thereby get used in FieldExtractExec plan = plan.transformDown(EsRelation.class, esr -> { @@ -1149,21 +1134,12 @@ private LogicalPlan doRule(LogicalPlan plan) { return plan; } - private Expression resolveAttribute(UnresolvedAttribute ua, Map resolved) { - var named = resolveAgainstList(ua, resolved.keySet()); - return switch (named.size()) { - case 0 -> ua; - case 1 -> named.get(0).equals(ua) ? ua : resolved.get(named.get(0)); - default -> ua.withUnresolvedMessage("Resolved [" + ua + "] unexpectedly to multiple attributes " + named); - }; - } - private Expression resolveConvertFunction(AbstractConvertFunction convert, List unionFieldAttributes) { if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { HashMap typeResolutions = new HashMap<>(); Set supportedTypes = convert.supportedTypes(); - imf.getTypesToIndices().keySet().forEach(typeName -> { - DataType type = DataType.fromTypeName(typeName); + imf.types().forEach(type -> { + // TODO: Shouldn't we perform widening of small numerical types here? if (supportedTypes.contains(type)) { TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); var concreteConvert = typeSpecificConvert(convert, fa.source(), type, imf); @@ -1236,13 +1212,10 @@ private Expression typeSpecificConvert(AbstractConvertFunction convert, Source s */ private static class UnionTypesCleanup extends Rule { public LogicalPlan apply(LogicalPlan plan) { - LogicalPlan planWithCheckedUnionTypes = plan.transformUp(LogicalPlan.class, p -> { - if (p instanceof EsRelation esRelation) { - // Leave esRelation as InvalidMappedField so that UNSUPPORTED fields can still pass through - return esRelation; - } - return p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved); - }); + LogicalPlan planWithCheckedUnionTypes = plan.transformUp( + LogicalPlan.class, + p -> p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved) + ); // To drop synthetic attributes at the end, we need to compute the plan's output. // This is only legal to do if the plan is resolved. @@ -1254,7 +1227,14 @@ public LogicalPlan apply(LogicalPlan plan) { static Attribute checkUnresolved(FieldAttribute fa) { if (fa.field() instanceof InvalidMappedField imf) { String unresolvedMessage = "Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage(); - return new UnresolvedAttribute(fa.source(), fa.name(), fa.id(), unresolvedMessage, null); + String types = imf.getTypesToIndices().keySet().stream().collect(Collectors.joining(",")); + return new UnsupportedAttribute( + fa.source(), + fa.name(), + new UnsupportedEsField(imf.getName(), types), + unresolvedMessage, + fa.id() + ); } return fa; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index bbfff5acd92b8..374cc17b7b902 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -111,16 +111,23 @@ else if (p.resolved()) { } e.forEachUp(ae -> { - // we're only interested in the children + // Special handling for Project and unsupported/union types: disallow renaming them but pass them through otherwise. + if (p instanceof Project) { + if (ae instanceof Alias as && as.child() instanceof UnsupportedAttribute ua) { + failures.add(fail(ae, ua.unresolvedMessage())); + } + if (ae instanceof UnsupportedAttribute) { + return; + } + } + + // Do not fail multiple times in case the children are already unresolved. if (ae.childrenResolved() == false) { return; } if (ae instanceof Unresolvable u) { - // special handling for Project and unsupported types - if (p instanceof Project == false || u instanceof UnsupportedAttribute == false) { - failures.add(fail(ae, u.unresolvedMessage())); - } + failures.add(fail(ae, u.unresolvedMessage())); } if (ae.typeResolved().unresolved()) { failures.add(fail(ae, ae.typeResolved().message())); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java index 9f8e36ba81b15..e35ca7e518532 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java @@ -132,6 +132,11 @@ public boolean analyzed() { return analyzed; } + @Override + protected TypeResolution resolveType() { + return new TypeResolution("unresolved function [" + name + "]"); + } + @Override public DataType dataType() { throw new UnresolvedException("dataType", this); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index b731a400deba3..cf97558cd2676 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -36,7 +36,7 @@ import java.util.Set; import java.util.function.Function; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isTypeOrUnionType; /** * Base class for functions that converts a field into a function-specific type. @@ -76,14 +76,14 @@ protected final TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - return isType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(supportedTypes())); + return isTypeOrUnionType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(supportedTypes())); } public Set supportedTypes() { return factories().keySet(); } - public static String supportedTypesNames(Set types) { + private static String supportedTypesNames(Set types) { List supportedTypesNames = new ArrayList<>(types.size()); HashSet supportTypes = new HashSet<>(types); if (supportTypes.containsAll(NUMERIC_TYPES)) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java index 1dde8e9e95990..35d5229d4e52f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java @@ -21,11 +21,6 @@ public interface Stats { */ Stats with(List newGroupings, List newAggregates); - /** - * Have all the expressions in this plan been resolved? - */ - boolean expressionsResolved(); - /** * List containing both the aggregate expressions and grouping expressions. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index f449139047851..583251817d681 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -632,30 +632,6 @@ public void testRenameReuseAlias() { """, "_meta_field", "e", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary"); } - public void testRenameUnsupportedField() { - assertProjectionWithMapping(""" - from test - | rename unsupported as u - | keep int, u, float - """, "mapping-multi-field-variation.json", "int", "u", "float"); - } - - public void testRenameUnsupportedFieldChained() { - assertProjectionWithMapping(""" - from test - | rename unsupported as u1, u1 as u2 - | keep int, u2, float - """, "mapping-multi-field-variation.json", "int", "u2", "float"); - } - - public void testRenameUnsupportedAndResolved() { - assertProjectionWithMapping(""" - from test - | rename unsupported as u, float as f - | keep int, u, f - """, "mapping-multi-field-variation.json", "int", "u", "f"); - } - public void testRenameUnsupportedSubFieldAndResolved() { assertProjectionWithMapping(""" from test @@ -1540,7 +1516,7 @@ public void testEnrichWrongMatchFieldType() { | enrich languages on x | keep first_name, language_name, id """)); - assertThat(e.getMessage(), containsString("Unsupported type [BOOLEAN] for enrich matching field [x]; only [KEYWORD,")); + assertThat(e.getMessage(), containsString("Unsupported type [boolean] for enrich matching field [x]; only [keyword, ")); e = expectThrows(VerificationException.class, () -> analyze(""" FROM airports @@ -1548,7 +1524,7 @@ public void testEnrichWrongMatchFieldType() { | ENRICH city_boundaries ON x | KEEP abbrev, airport, region """, "airports", "mapping-airports.json")); - assertThat(e.getMessage(), containsString("Unsupported type [KEYWORD] for enrich matching field [x]; only [GEO_POINT,")); + assertThat(e.getMessage(), containsString("Unsupported type [keyword] for enrich matching field [x]; only [geo_point, ")); } public void testValidEnrich() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 53b2fdc329387..06da10dbbea2f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -12,12 +12,21 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.parser.QueryParams; import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; @@ -46,6 +55,160 @@ public void testIncompatibleTypesInMathOperation() { ); } + public void testUnsupportedAndMultiTypedFields() { + final String unsupported = "unsupported"; + final String multiTyped = "multi_typed"; + + EsField unsupportedField = new UnsupportedEsField(unsupported, "flattened"); + // Use linked maps/sets to fix the order in the error message. + LinkedHashSet ipIndices = new LinkedHashSet<>(); + ipIndices.add("test1"); + ipIndices.add("test2"); + LinkedHashMap> typesToIndices = new LinkedHashMap<>(); + typesToIndices.put("ip", ipIndices); + typesToIndices.put("keyword", Set.of("test3")); + EsField multiTypedField = new InvalidMappedField(multiTyped, typesToIndices); + + // Also add an unsupported/multityped field under the names `int` and `double` so we can use `LOOKUP int_number_names ...` and + // `LOOKUP double_number_names` without renaming the fields first. + IndexResolution indexWithUnsupportedAndMultiTypedField = IndexResolution.valid( + new EsIndex( + "test*", + Map.of(unsupported, unsupportedField, multiTyped, multiTypedField, "int", unsupportedField, "double", multiTypedField) + ) + ); + Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnsupportedAndMultiTypedField); + + assertEquals( + "1:22: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | dissect unsupported \"%{foo}\"", analyzer) + ); + assertEquals( + "1:22: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | dissect multi_typed \"%{foo}\"", analyzer) + ); + + assertEquals( + "1:19: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | grok unsupported \"%{WORD:foo}\"", analyzer) + ); + assertEquals( + "1:19: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | grok multi_typed \"%{WORD:foo}\"", analyzer) + ); + + assertEquals( + "1:36: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | enrich client_cidr on unsupported", analyzer) + ); + assertEquals( + "1:36: Unsupported type [unsupported] for enrich matching field [multi_typed];" + + " only [keyword, text, ip, long, integer, float, double, datetime] allowed for type [range]", + error("from test* | enrich client_cidr on multi_typed", analyzer) + ); + + assertEquals( + "1:23: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | eval x = unsupported", analyzer) + ); + assertEquals( + "1:23: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | eval x = multi_typed", analyzer) + ); + + assertEquals( + "1:32: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | eval x = to_lower(unsupported)", analyzer) + ); + assertEquals( + "1:32: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | eval x = to_lower(multi_typed)", analyzer) + ); + + assertEquals( + "1:32: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | stats count(1) by unsupported", analyzer) + ); + assertEquals( + "1:32: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | stats count(1) by multi_typed", analyzer) + ); + assertEquals( + "1:38: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | inlinestats count(1) by unsupported", analyzer) + ); + assertEquals( + "1:38: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | inlinestats count(1) by multi_typed", analyzer) + ); + + assertEquals( + "1:27: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | stats values(unsupported)", analyzer) + ); + assertEquals( + "1:27: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | stats values(multi_typed)", analyzer) + ); + assertEquals( + "1:33: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | inlinestats values(unsupported)", analyzer) + ); + assertEquals( + "1:33: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | inlinestats values(multi_typed)", analyzer) + ); + + assertEquals( + "1:27: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | stats values(unsupported)", analyzer) + ); + assertEquals( + "1:27: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | stats values(multi_typed)", analyzer) + ); + + // LOOKUP with unsupported type + assertEquals( + "1:41: column type mismatch, table column was [integer] and original column was [unsupported]", + error("from test* | lookup int_number_names on int", analyzer) + ); + // LOOKUP with multi-typed field + assertEquals( + "1:44: column type mismatch, table column was [double] and original column was [unsupported]", + error("from test* | lookup double_number_names on double", analyzer) + ); + + assertEquals( + "1:24: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | mv_expand unsupported", analyzer) + ); + assertEquals( + "1:24: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | mv_expand multi_typed", analyzer) + ); + + assertEquals( + "1:21: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | rename unsupported as x", analyzer) + ); + assertEquals( + "1:21: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | rename multi_typed as x", analyzer) + ); + } + public void testRoundFunctionInvalidInputs() { assertEquals( "1:31: first argument of [round(b, 3)] must be [numeric], found value [b] type [keyword]", diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml index 003b1d0651d11..92b3f4d1b084d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml @@ -296,12 +296,44 @@ load two indices, showing unsupported type and null value for event_duration: --- load two indices with no conversion function, but needs TO_LONG conversion: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [union_types_fix_rename_resolution] + reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP" + - do: - catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword\], \[long\] in \[events_ip_long\]/' + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events_ip_* METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "ip" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "unsupported" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 14 } + - match: { values.0.0: "events_ip_keyword" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: null } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_ip_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: "172.21.3.15" } + - match: { values.7.3: null } + - match: { values.7.4: "Connected to 10.1.0.1" } + --- load two indices with incorrect conversion function, TO_IP instead of TO_LONG: - do: @@ -450,12 +482,44 @@ load two indices, showing unsupported type and null value for client_ip: --- load two indices with no conversion function, but needs TO_IP conversion: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [union_types_fix_rename_resolution] + reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP" + - do: - catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_long\], \[keyword\] in \[events_keyword_long\]/' + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events_*_long METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "unsupported" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 14 } + - match: { values.0.0: "events_ip_long" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: null } + - match: { values.0.3: 1756467 } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_keyword_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: null } + - match: { values.7.3: 1756467 } + - match: { values.7.4: "Connected to 10.1.0.1" } + --- load two indices with incorrect conversion function, TO_LONG instead of TO_IP: - do: @@ -629,20 +693,103 @@ load two indexes, convert client_ip and group by something invalid: --- load four indices with single conversion function TO_LONG: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [union_types_fix_rename_resolution] + reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP" + - do: - catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_keyword, events_ip_long\], \[keyword\] in \[events_keyword_keyword, events_keyword_long\]/' + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events_* METADATA _index | EVAL event_duration = TO_LONG(event_duration) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "unsupported" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 28 } + - match: { values.0.0: "events_ip_keyword" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: null } + - match: { values.0.3: 1756467 } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_ip_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: null } + - match: { values.7.3: 1756467 } + - match: { values.7.4: "Connected to 10.1.0.1" } + - match: { values.14.0: "events_keyword_keyword" } + - match: { values.14.1: "2023-10-23T13:55:01.543Z" } + - match: { values.14.2: null } + - match: { values.14.3: 1756467 } + - match: { values.14.4: "Connected to 10.1.0.1" } + - match: { values.21.0: "events_keyword_long" } + - match: { values.21.1: "2023-10-23T13:55:01.543Z" } + - match: { values.21.2: null } + - match: { values.21.3: 1756467 } + - match: { values.21.4: "Connected to 10.1.0.1" } + --- load four indices with single conversion function TO_IP: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [union_types_fix_rename_resolution] + reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP" + - do: - catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword, events_keyword_keyword\], \[long\] in \[events_ip_long, events_keyword_long\]/' + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM events_* METADATA _index | EVAL client_ip = TO_IP(client_ip) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "ip" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "unsupported" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 28 } + - match: { values.0.0: "events_ip_keyword" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: null } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_ip_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: "172.21.3.15" } + - match: { values.7.3: null } + - match: { values.7.4: "Connected to 10.1.0.1" } + - match: { values.14.0: "events_keyword_keyword" } + - match: { values.14.1: "2023-10-23T13:55:01.543Z" } + - match: { values.14.2: "172.21.3.15" } + - match: { values.14.3: null } + - match: { values.14.4: "Connected to 10.1.0.1" } + - match: { values.21.0: "events_keyword_long" } + - match: { values.21.1: "2023-10-23T13:55:01.543Z" } + - match: { values.21.2: "172.21.3.15" } + - match: { values.21.3: null } + - match: { values.21.4: "Connected to 10.1.0.1" } --- load four indices with multiple conversion functions TO_LONG and TO_IP: - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 8f291600acbf6..642407ac6d45b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -253,12 +253,24 @@ from index pattern unsupported counter: --- from index pattern explicit counter use: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [union_types_fix_rename_resolution] + reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP" + - do: - catch: '/Cannot use field \[k8s.pod.network.tx\] due to ambiguities being mapped as different metric types in indices: \[test, test2\]/' + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'FROM test* | keep *.tx' - + - match: {columns.0.name: "k8s.pod.network.tx"} + - match: {columns.0.type: "unsupported"} + - length: {values: 10} --- _source: From d99f87121c2a6ff2461ed56ee7418a8d665ed781 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 9 Aug 2024 11:07:14 +0200 Subject: [PATCH 17/37] Fix remote cluster credential secure settings reload (#111535) Due to the `cluster:admin/xpack/security` action name prefix, the internal action `cluster:admin/xpack/security/remote_cluster_credentials/reload` to reload remote cluster credentials fails for users that have the `manage` cluster privilege. This does not align with our documentation and the overall permission requirements for reloading secure settings. This PR renames the action to match the `manage` privilege. Since this is a local-only action there are no BWC concerns with this rename. Fixes: https://github.com/elastic/elasticsearch/issues/111543 --- docs/changelog/111535.yaml | 5 ++ ...nsportNodesReloadSecureSettingsAction.java | 1 + .../core/security/action/ActionTypes.java | 7 ++- ...AbstractRemoteClusterSecurityTestCase.java | 5 +- ...lusterSecurityReloadCredentialsRestIT.java | 30 ++++++++++++ .../src/javaRestTest/resources/roles.yml | 3 ++ .../xpack/security/operator/Constants.java | 2 +- .../xpack/security/SecurityTests.java | 15 ++++-- ...gsWithPasswordProtectedKeystoreRestIT.java | 46 +++++++++++++++++++ .../src/javaRestTest/resources/roles.yml | 12 +++++ 10 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/111535.yaml diff --git a/docs/changelog/111535.yaml b/docs/changelog/111535.yaml new file mode 100644 index 0000000000000..4beebbf28d4e1 --- /dev/null +++ b/docs/changelog/111535.yaml @@ -0,0 +1,5 @@ +pr: 111535 +summary: Fix remote cluster credential secure settings reload +area: Authorization +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index f906b7d659b7b..82df12d9cfef7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -123,6 +123,7 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation( final List exceptions = new ArrayList<>(); // broadcast the new settings object (with the open embedded keystore) to all reloadable plugins pluginsService.filterPlugins(ReloadablePlugin.class).forEach(p -> { + logger.debug("Reloading plugin [" + p.getClass().getSimpleName() + "]"); try { p.reload(settingsWithKeystore); } catch (final Exception e) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java index 52f8c7cf456d9..bf9484c52800a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java @@ -20,8 +20,13 @@ public final class ActionTypes { private ActionTypes() {}; + // Note: this action is *not* prefixed with `cluster:admin/xpack/security` since it would otherwise be excluded from the `manage` + // privilege -- instead it matches its prefix to `TransportNodesReloadSecureSettingsAction` which is the "parent" transport action + // that invokes the overall reload flow. + // This allows us to maintain the invariant that the parent reload secure settings action can be executed with the `manage` privilege + // without trappy system-context switches. public static final ActionType RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION = new ActionType<>( - "cluster:admin/xpack/security/remote_cluster_credentials/reload" + "cluster:admin/nodes/reload_secure_settings/security/remote_cluster_credentials" ); public static final ActionType QUERY_USER_ACTION = new ActionType<>("cluster:admin/xpack/security/user/query"); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java index c4a058013caf2..2c61707f57a5a 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java @@ -48,11 +48,12 @@ public abstract class AbstractRemoteClusterSecurityTestCase extends ESRestTestCa protected static final String USER = "test_user"; protected static final SecureString PASS = new SecureString("x-pack-test-password".toCharArray()); protected static final String REMOTE_SEARCH_USER = "remote_search_user"; + protected static final String MANAGE_USER = "manage_user"; protected static final String REMOTE_METRIC_USER = "remote_metric_user"; protected static final String REMOTE_TRANSFORM_USER = "remote_transform_user"; protected static final String REMOTE_SEARCH_ROLE = "remote_search"; protected static final String REMOTE_CLUSTER_ALIAS = "my_remote_cluster"; - private static final String KEYSTORE_PASSWORD = "keystore-password"; + static final String KEYSTORE_PASSWORD = "keystore-password"; protected static LocalClusterConfigProvider commonClusterConfig = cluster -> cluster.module("analysis-common") .keystorePassword(KEYSTORE_PASSWORD) @@ -217,7 +218,7 @@ protected void removeRemoteClusterCredentials(String clusterAlias, MutableSettin } @SuppressWarnings("unchecked") - private void reloadSecureSettings() throws IOException { + protected void reloadSecureSettings() throws IOException { final Request request = new Request("POST", "/_nodes/reload_secure_settings"); request.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); final Response reloadResponse = adminClient().performRequest(request); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java index a721605d228db..42982e6183613 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java @@ -34,7 +34,12 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; // account for slow stored secure settings updates (involves removing and re-creating the keystore) @TimeoutSuite(millis = 10 * TimeUnits.MINUTE) @@ -78,6 +83,7 @@ public class RemoteClusterSecurityReloadCredentialsRestIT extends AbstractRemote }) .rolesFile(Resource.fromClasspath("roles.yml")) .user(REMOTE_SEARCH_USER, PASS.toString(), "read_remote_shared_logs", false) + .user(MANAGE_USER, PASS.toString(), "manage_role", false) .build(); } @@ -237,4 +243,28 @@ private Response performRequestWithRemoteSearchUser(final Request request) throw return client().performRequest(request); } + @Override + @SuppressWarnings("unchecked") + protected void reloadSecureSettings() throws IOException { + final Request request = new Request("POST", "/_nodes/reload_secure_settings"); + request.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); + // execute as user with only `manage` cluster privilege + final Response reloadResponse = performRequestWithManageUser(request); + assertOK(reloadResponse); + final Map map = entityAsMap(reloadResponse); + assertThat(map.get("nodes"), instanceOf(Map.class)); + final Map nodes = (Map) map.get("nodes"); + assertThat(nodes, is(not(anEmptyMap()))); + for (Map.Entry entry : nodes.entrySet()) { + assertThat(entry.getValue(), instanceOf(Map.class)); + final Map node = (Map) entry.getValue(); + assertThat(node.get("reload_exception"), nullValue()); + } + } + + private Response performRequestWithManageUser(final Request request) throws IOException { + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(MANAGE_USER, PASS))); + return client().performRequest(request); + } + } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml index d0487f378d34e..b61daa068ed1a 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml @@ -38,3 +38,6 @@ ccr_user_role: - names: [ 'leader-index', 'shared-*', 'metrics-*' ] privileges: [ 'cross_cluster_replication' ] clusters: [ "*" ] + +manage_role: + cluster: [ 'manage' ] diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 04ab277fb09bf..c5304d8313df2 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -52,6 +52,7 @@ public class Constants { "cluster:admin/migration/get_system_feature", "cluster:admin/migration/post_system_feature", "cluster:admin/nodes/reload_secure_settings", + "cluster:admin/nodes/reload_secure_settings/security/remote_cluster_credentials", "cluster:admin/persistent/completion", "cluster:admin/persistent/remove", "cluster:admin/persistent/start", @@ -276,7 +277,6 @@ public class Constants { "cluster:admin/xpack/security/profile/suggest", "cluster:admin/xpack/security/profile/set_enabled", "cluster:admin/xpack/security/realm/cache/clear", - "cluster:admin/xpack/security/remote_cluster_credentials/reload", "cluster:admin/xpack/security/role/delete", "cluster:admin/xpack/security/role/get", "cluster:admin/xpack/security/role/query", diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 400bc35b93fd5..a07a7a3a5dd27 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionModule; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -152,7 +151,6 @@ import static org.mockito.Mockito.when; public class SecurityTests extends ESTestCase { - private Security security = null; private ThreadContext threadContext = null; private SecurityContext securityContext = null; @@ -960,8 +958,11 @@ public List loadExtensions(Class extensionPointType) { public void testReload() throws Exception { final Settings settings = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir()).build(); - final PlainActionFuture value = new PlainActionFuture<>(); + final ThreadPool threadPool = mock(ThreadPool.class); + threadContext = new ThreadContext(Settings.EMPTY); + when(threadPool.getThreadContext()).thenReturn(threadContext); final Client mockedClient = mock(Client.class); + when(mockedClient.threadPool()).thenReturn(threadPool); final JwtRealm mockedJwtRealm = mock(JwtRealm.class); final List reloadableComponents = List.of(mockedJwtRealm); @@ -992,11 +993,17 @@ protected List getReloadableSecurityComponents() { verify(mockedJwtRealm).reload(same(inputSettings)); } - public void testReloadWithFailures() throws Exception { + public void testReloadWithFailures() { final Settings settings = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir()).build(); final boolean failRemoteClusterCredentialsReload = randomBoolean(); + + final ThreadPool threadPool = mock(ThreadPool.class); + threadContext = new ThreadContext(Settings.EMPTY); + when(threadPool.getThreadContext()).thenReturn(threadContext); final Client mockedClient = mock(Client.class); + when(mockedClient.threadPool()).thenReturn(threadPool); + final JwtRealm mockedJwtRealm = mock(JwtRealm.class); final List reloadableComponents = List.of(mockedJwtRealm); if (failRemoteClusterCredentialsReload) { diff --git a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java index 0625ec166e32c..8e9dca396fb8e 100644 --- a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java +++ b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java @@ -7,7 +7,9 @@ package org.elasticsearch.password_protected_keystore; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -18,6 +20,7 @@ import org.elasticsearch.xcontent.ObjectPath; import org.junit.ClassRule; +import java.io.IOException; import java.util.Map; import static org.hamcrest.Matchers.anyOf; @@ -39,6 +42,7 @@ public class ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT extends ESR .name("javaRestTest") .keystore(nodeSpec -> Map.of("xpack.security.transport.ssl.secure_key_passphrase", "transport-password")) .setting("xpack.security.enabled", "true") + .setting("xpack.ml.enabled", "false") .setting("xpack.security.authc.anonymous.roles", "anonymous") .setting("xpack.security.transport.ssl.enabled", "true") .setting("xpack.security.transport.ssl.certificate", "transport.crt") @@ -50,6 +54,9 @@ public class ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT extends ESR .configFile("ca.crt", Resource.fromClasspath("ssl/ca.crt")) .user("admin_user", "admin-password") .user("test-user", "test-user-password", "user_role", false) + .user("manage-user", "test-user-password", "manage_role", false) + .user("manage-security-user", "test-user-password", "manage_security_role", false) + .user("monitor-user", "test-user-password", "monitor_role", false) .build(); @Override @@ -74,6 +81,33 @@ public void testReloadSecureSettingsWithCorrectPassword() throws Exception { } } + @SuppressWarnings("unchecked") + public void testReloadSecureSettingsWithDifferentPrivileges() throws Exception { + final Request request = new Request("POST", "/_nodes/reload_secure_settings"); + request.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); + final Response response = performRequestWithUser("manage-user", request); + final Map map = entityAsMap(response); + assertThat(ObjectPath.eval("cluster_name", map), equalTo("javaRestTest")); + assertThat(map.get("nodes"), instanceOf(Map.class)); + final Map nodes = (Map) map.get("nodes"); + assertThat(nodes.size(), equalTo(NUM_NODES)); + for (Map.Entry entry : nodes.entrySet()) { + assertThat(entry.getValue(), instanceOf(Map.class)); + final Map node = (Map) entry.getValue(); + assertThat(node.get("reload_exception"), nullValue()); + } + expectThrows403(() -> { + final Request innerRequest = new Request("POST", "/_nodes/reload_secure_settings"); + innerRequest.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); + performRequestWithUser("manage-security-user", innerRequest); + }); + expectThrows403(() -> { + final Request innerRequest = new Request("POST", "/_nodes/reload_secure_settings"); + innerRequest.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); + performRequestWithUser("monitor-user", request); + }); + } + @SuppressWarnings("unchecked") public void testReloadSecureSettingsWithIncorrectPassword() throws Exception { final Request request = new Request("POST", "_nodes/reload_secure_settings"); @@ -136,4 +170,16 @@ protected Settings restAdminSettings() { String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray())); return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); } + + private static void expectThrows403(ThrowingRunnable runnable) { + assertThat(expectThrows(ResponseException.class, runnable).getResponse().getStatusLine().getStatusCode(), equalTo(403)); + } + + private Response performRequestWithUser(final String username, final Request request) throws IOException { + request.setOptions( + RequestOptions.DEFAULT.toBuilder() + .addHeader("Authorization", basicAuthHeaderValue(username, new SecureString("test-user-password"))) + ); + return client().performRequest(request); + } } diff --git a/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml b/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml index 60dd459d0f824..c551d8200bfe0 100644 --- a/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml +++ b/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml @@ -2,3 +2,15 @@ user_role: cluster: [ALL] indices: [] + +manage_role: + cluster: [MANAGE] + indices: [] + +manage_security_role: + cluster: [MANAGE_SECURITY] + indices: [] + +monitor_role: + cluster: [MONITOR] + indices: [] From 5a7a032cead75dba6a664555ab4295dcdd562a56 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 9 Aug 2024 11:12:46 +0200 Subject: [PATCH 18/37] [ML] Force time shift documentation (#111668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: István Zoltán Szabó --- .../anomaly-detection/apis/put-job.asciidoc | 14 +++++++++++ .../apis/update-job.asciidoc | 23 +++++++++++++++---- docs/reference/ml/ml-shared.asciidoc | 14 +++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index 012904a9affa7..7bf02e7a0dd6e 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -105,6 +105,20 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules] (array) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] +//Begin analysis_config.detectors.custom_rules.params +`params`::: +(object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params] ++ +.Properties of `params` +[%collapsible%open] +======= +`force_time_shift`:::: +(object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params-force-time-shift] +======= +//End analysis_config.detectors.custom_rules.params + //Begin analysis_config.detectors.custom_rules.conditions `conditions`::: (array) diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc index 6953235c854cb..ee13247fc8838 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc @@ -15,7 +15,7 @@ Updates certain properties of an {anomaly-job}. [[ml-update-job-prereqs]] == {api-prereq-title} -Requires the `manage_ml` cluster privilege. This privilege is included in the +Requires the `manage_ml` cluster privilege. This privilege is included in the `machine_learning_admin` built-in role. [[ml-update-job-path-parms]] @@ -51,7 +51,7 @@ You can update the `analysis_limits` only while the job is closed. [%collapsible%open] ==== `model_memory_limit`::: -(long or string) +(long or string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] + -- @@ -61,8 +61,8 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] determine the current usage, refer to the `model_bytes` value in the <> API. * If the `memory_status` property in the -<> has a value of -`hard_limit`, this means that it was unable to process some data. You might want +<> has a value of +`hard_limit`, this means that it was unable to process some data. You might want to re-run the job with an increased `model_memory_limit`. ======= -- @@ -111,6 +111,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules] (array) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] +//Begin analysis_config.detectors.custom_rules.params +`params`::: +(object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params] ++ +.Properties of `params` +[%collapsible%open] +======= +[[force-time-shift-params]] +`force_time_shift`:::: +(object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params-force-time-shift] +======= +//End analysis_config.detectors.custom_rules.params + // Begin detectors.custom_rules.conditions `conditions`::: (array) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 15a994115c88c..44c2012f502e1 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -384,6 +384,10 @@ model. Unless you also specify `skip_result`, the results will be created as usual. This action is suitable when certain values are expected to be consistently anomalous and they affect the model in a way that negatively impacts the rest of the results. +* `force_time_shift`: This action will shift the time inside the anomaly detector by a specified +amount. This is useful, e.g. to quickly adapt to the daylight saving time events that +are known beforehand. This action requires a `force_time_shift` parameter +in the `params` object. end::custom-rules-actions[] tag::custom-rules-scope[] @@ -428,6 +432,16 @@ tag::custom-rules-conditions-value[] The value that is compared against the `applies_to` field using the `operator`. end::custom-rules-conditions-value[] +tag::custom-rules-params[] +A set of parameter objects that customize the actions defined in the custom rules +actions array. The available parameters (depending on the specified actions) include: +`force_time_shift`. +end::custom-rules-params[] + +tag::custom-rules-params-force-time-shift[] +Set `time_shift_amount` to the signed number of seconds by which you want to shift the time. +end::custom-rules-params-force-time-shift[] + tag::custom-settings[] Advanced configuration option. Contains custom metadata about the job. For example, it can contain custom URL information as shown in From 843a6e44e3087208d1d31bc91faeeb8c6313cc4c Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 9 Aug 2024 11:34:32 +0200 Subject: [PATCH 19/37] ESQL: Add tests for sort, where with unsupported type (#111737) --- .../xpack/esql/analysis/VerifierTests.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 06da10dbbea2f..6e9cda60f7d5d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -207,6 +207,26 @@ public void testUnsupportedAndMultiTypedFields() { + " [ip] in [test1, test2], [keyword] in [test3]", error("from test* | rename multi_typed as x", analyzer) ); + + assertEquals( + "1:19: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | sort unsupported asc", analyzer) + ); + assertEquals( + "1:19: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | sort multi_typed desc", analyzer) + ); + + assertEquals( + "1:20: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | where unsupported is null", analyzer) + ); + assertEquals( + "1:20: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | where multi_typed is not null", analyzer) + ); } public void testRoundFunctionInvalidInputs() { From d19d2509fcf0381e9735c8f811463787baf106f1 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:30:39 +0200 Subject: [PATCH 20/37] [ML] Force time shift integration test (#111620) This PR adds an integration test to verify that the force_time_shift detection rule works as expected. --- .../ml/integration/DetectionRulesIT.java | 67 +++++++++++++++++++ .../MlNativeAutodetectIntegTestCase.java | 21 ++++++ 2 files changed, 88 insertions(+) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java index fec85730aaf2b..0effe5349d43a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; import org.elasticsearch.xpack.core.ml.action.UpdateFilterAction; +import org.elasticsearch.xpack.core.ml.annotations.Annotation; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; @@ -20,7 +21,10 @@ import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.config.Operator; +import org.elasticsearch.xpack.core.ml.job.config.RuleAction; import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; +import org.elasticsearch.xpack.core.ml.job.config.RuleParams; +import org.elasticsearch.xpack.core.ml.job.config.RuleParamsForForceTimeShift; import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; @@ -39,7 +43,9 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.oneOf; /** @@ -299,6 +305,67 @@ public void testScopeAndCondition() throws IOException { assertThat(records.get(0).getOverFieldValue(), equalTo("222.222.222.222")); } + public void testForceTimeShiftAction() throws Exception { + // The test ensures that the force time shift action works as expected. + + long timeShiftAmount = 3600L; + long timestampStartMillis = 1491004800000L; + long bucketSpanMillis = 3600000L; + long timeShiftTimestamp = (timestampStartMillis + bucketSpanMillis) / 1000; + + int totalBuckets = 2 * 24; + + DetectionRule rule = new DetectionRule.Builder( + Arrays.asList(new RuleCondition(RuleCondition.AppliesTo.TIME, Operator.GTE, timeShiftTimestamp)) + ).setActions(RuleAction.FORCE_TIME_SHIFT).setParams(new RuleParams(new RuleParamsForForceTimeShift(timeShiftAmount))).build(); + + Detector.Builder detector = new Detector.Builder("mean", "value"); + detector.setRules(Arrays.asList(rule)); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Arrays.asList(detector.build())); + analysisConfig.setBucketSpan(TimeValue.timeValueMillis(bucketSpanMillis)); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + Job.Builder job = new Job.Builder("detection-rules-it-test-force-time-shift"); + job.setAnalysisConfig(analysisConfig); + job.setDataDescription(dataDescription); + + putJob(job); + openJob(job.getId()); + + // post some data + int normalValue = 400; + List data = new ArrayList<>(); + long timestamp = timestampStartMillis; + for (int bucket = 0; bucket < totalBuckets; bucket++) { + Map record = new HashMap<>(); + record.put("time", timestamp); + record.put("value", normalValue); + data.add(createJsonRecord(record)); + timestamp += bucketSpanMillis; + } + + postData(job.getId(), joinBetween(0, data.size(), data)); + closeJob(job.getId()); + + List annotations = getAnnotations(); + assertThat(annotations.size(), greaterThanOrEqualTo(1)); + assertThat(annotations.size(), lessThanOrEqualTo(3)); + + // Check that annotation contain the expected time shift + boolean countingModelAnnotationFound = false; + boolean individualModelAnnotationFound = false; + for (Annotation annotation : annotations) { + if (annotation.getAnnotation().contains("Counting model shifted time by")) { + countingModelAnnotationFound = true; + assertThat(annotation.getAnnotation(), containsString(timeShiftAmount + " seconds")); + } else if (annotation.getAnnotation().contains("Model shifted time by")) { + individualModelAnnotationFound = true; + assertThat(annotation.getAnnotation(), containsString(timeShiftAmount + " seconds")); + } + } + assertThat("Counting model annotation with time shift not found", countingModelAnnotationFound, equalTo(true)); + assertThat("Individual model annotation with time shift not found", individualModelAnnotationFound, equalTo(true)); + } + private String createIpRecord(long timestamp, String ip) throws IOException { Map record = new HashMap<>(); record.put("time", timestamp); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java index ce265d0e895aa..2e096f3262cb6 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java @@ -287,6 +287,27 @@ protected void assertThatNumberOfAnnotationsIsEqualTo(int expectedNumberOfAnnota }); } + protected List getAnnotations() throws Exception { + List annotations = new ArrayList<>(); + // Refresh the annotations index so that recently indexed annotation docs are visible. + indicesAdmin().prepareRefresh(AnnotationIndex.LATEST_INDEX_NAME) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .get(); + + SearchRequest searchRequest = new SearchRequest(AnnotationIndex.READ_ALIAS_NAME).indicesOptions( + IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN + ); + assertCheckedResponse(client().search(searchRequest), searchResponse -> { + + for (SearchHit hit : searchResponse.getHits().getHits()) { + try (XContentParser parser = createParser(jsonXContent, hit.getSourceRef())) { + annotations.add(Annotation.fromXContent(parser, null)); + } + } + }); + return annotations; + } + protected ForecastRequestStats getForecastStats(String jobId, String forecastId) throws Exception { SetOnce forecastRequestStats = new SetOnce<>(); assertCheckedResponse( From 8bf0850b9bb3bd9c521197932dfe080043caf25e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 9 Aug 2024 23:40:52 +1000 Subject: [PATCH 21/37] Mute org.elasticsearch.xpack.esql.analysis.VerifierTests testUnsupportedAndMultiTypedFields #111753 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e439759d75dab..1fc04d84e3173 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -125,6 +125,9 @@ tests: - class: org.elasticsearch.tdigest.ComparisonTests method: testSparseGaussianDistribution issue: https://github.com/elastic/elasticsearch/issues/111721 +- class: org.elasticsearch.xpack.esql.analysis.VerifierTests + method: testUnsupportedAndMultiTypedFields + issue: https://github.com/elastic/elasticsearch/issues/111753 # Examples: # From 4694a44bd4ac042a0d9f047d376f46361fc01f2a Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 9 Aug 2024 15:48:27 +0200 Subject: [PATCH 22/37] Always enforce strict role validation (#111056) Updates role and API key related request translation interfaces to remove restriction parameters. These are no longer used downstream. --- .../apikey/CreateApiKeyRequestBuilderFactory.java | 5 ++--- .../GetBuiltinPrivilegesResponseTranslator.java | 5 ++--- .../core/security/action/role/PutRoleRequest.java | 9 --------- .../action/role/PutRoleRequestBuilderFactory.java | 6 ++---- .../rest/action/apikey/RestCreateApiKeyAction.java | 3 +-- .../privilege/RestGetBuiltinPrivilegesAction.java | 3 +-- .../rest/action/role/RestPutRoleAction.java | 3 +-- .../test/api_key/40_view_role_descriptors.yml | 4 ++-- .../rest-api-spec/test/api_key/60_admin_user.yml | 14 +++++++------- 9 files changed, 18 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java index ff5592e339634..cfdf8147d8439 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java @@ -10,12 +10,11 @@ import org.elasticsearch.client.internal.Client; public interface CreateApiKeyRequestBuilderFactory { - CreateApiKeyRequestBuilder create(Client client, boolean restrictRequest); + CreateApiKeyRequestBuilder create(Client client); class Default implements CreateApiKeyRequestBuilderFactory { @Override - public CreateApiKeyRequestBuilder create(Client client, boolean restrictRequest) { - assert false == restrictRequest; + public CreateApiKeyRequestBuilder create(Client client) { return new CreateApiKeyRequestBuilder(client); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java index 2d018ae2f1b2f..0a1f49ffa04ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java @@ -9,11 +9,10 @@ public interface GetBuiltinPrivilegesResponseTranslator { - GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response, boolean restrictResponse); + GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response); class Default implements GetBuiltinPrivilegesResponseTranslator { - public GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response, boolean restrictResponse) { - assert false == restrictResponse; + public GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response) { return response; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java index 27f7c42d74018..46ba00a4f2768 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java @@ -45,7 +45,6 @@ public class PutRoleRequest extends ActionRequest { private Map metadata; private List remoteIndicesPrivileges = new ArrayList<>(); private RemoteClusterPermissions remoteClusterPermissions = RemoteClusterPermissions.NONE; - private boolean restrictRequest = false; private String description; public PutRoleRequest() {} @@ -84,14 +83,6 @@ public void addRemoteIndex(RoleDescriptor.RemoteIndicesPrivileges... privileges) remoteIndicesPrivileges.addAll(Arrays.asList(privileges)); } - public void restrictRequest(boolean restrictRequest) { - this.restrictRequest = restrictRequest; - } - - public boolean restrictRequest() { - return restrictRequest; - } - public void putRemoteCluster(RemoteClusterPermissions remoteClusterPermissions) { this.remoteClusterPermissions = remoteClusterPermissions; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java index 169bd5d4cc1f7..e8c5b465ace36 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java @@ -10,13 +10,11 @@ import org.elasticsearch.client.internal.Client; public interface PutRoleRequestBuilderFactory { - PutRoleRequestBuilder create(Client client, boolean restrictRequest); + PutRoleRequestBuilder create(Client client); class Default implements PutRoleRequestBuilderFactory { @Override - public PutRoleRequestBuilder create(Client client, boolean restrictRequest) { - // by default, we don't apply extra restrictions to Put Role requests and don't require checks against file-based roles - // these dependencies are only used by our stateless implementation + public PutRoleRequestBuilder create(Client client) { return new PutRoleRequestBuilder(client); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java index 217afdb3cfea2..29ffc76425060 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java @@ -55,8 +55,7 @@ public String getName() { @Override protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { - CreateApiKeyRequestBuilder builder = builderFactory.create(client, request.hasParam(RestRequest.PATH_RESTRICTED)) - .source(request.requiredContent(), request.getXContentType()); + CreateApiKeyRequestBuilder builder = builderFactory.create(client).source(request.requiredContent(), request.getXContentType()); String refresh = request.param("refresh"); if (refresh != null) { builder.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(request.param("refresh"))); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java index d804559bba0ec..5f0657079e694 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java @@ -62,14 +62,13 @@ public String getName() { @Override public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { - final boolean restrictResponse = request.hasParam(RestRequest.PATH_RESTRICTED); return channel -> client.execute( GetBuiltinPrivilegesAction.INSTANCE, new GetBuiltinPrivilegesRequest(), new RestBuilderListener<>(channel) { @Override public RestResponse buildResponse(GetBuiltinPrivilegesResponse response, XContentBuilder builder) throws Exception { - final var translatedResponse = responseTranslator.translate(response, restrictResponse); + final var translatedResponse = responseTranslator.translate(response); builder.startObject(); builder.array("cluster", translatedResponse.getClusterPrivileges()); builder.array("index", translatedResponse.getIndexPrivileges()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java index 82596738e95a7..6a819c098e9f1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java @@ -55,8 +55,7 @@ public String getName() { @Override public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { - final boolean restrictRequest = request.hasParam(RestRequest.PATH_RESTRICTED); - final PutRoleRequestBuilder requestBuilder = builderFactory.create(client, restrictRequest) + final PutRoleRequestBuilder requestBuilder = builderFactory.create(client) .source(request.param("name"), request.requiredContent(), request.getXContentType()) .setRefreshPolicy(request.param("refresh")); return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml index a7fb00a052528..e9f8359e41ce3 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml @@ -21,7 +21,7 @@ setup: ], "applications": [ { - "application": "myapp", + "application": "apm", "privileges": ["*"], "resources": ["*"] } @@ -497,7 +497,7 @@ teardown: ], "applications" : [ { - "application" : "myapp", + "application" : "apm", "privileges" : [ "*" ], diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml index 1a85dda7379be..643287bad4833 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml @@ -15,14 +15,14 @@ setup: security.put_privileges: body: > { - "myapp": { + "apm": { "read": { - "application": "myapp", + "application": "apm", "name": "read", "actions": [ "data:read/*" ] }, "write": { - "application": "myapp", + "application": "apm", "name": "write", "actions": [ "data:write/*" ] } @@ -33,7 +33,7 @@ setup: teardown: - do: security.delete_privileges: - application: myapp + application: apm name: "read,write" ignore: 404 @@ -254,7 +254,7 @@ teardown: ], "applications": [ { - "application": "myapp", + "application": "apm", "privileges": ["read"], "resources": ["*"] } @@ -299,7 +299,7 @@ teardown: ], "application": [ { - "application" : "myapp", + "application" : "apm", "resources" : [ "*", "some-other-res" ], "privileges" : [ "data:read/me", "data:write/me" ] } @@ -324,7 +324,7 @@ teardown: } } } - match: { "application" : { - "myapp" : { + "apm" : { "*" : { "data:read/me" : true, "data:write/me" : false From f0d06ee380d932f71ab3f2a8f8d688ac9a58100a Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 9 Aug 2024 17:21:39 +0200 Subject: [PATCH 23/37] Skip LOOKUP/INLINESTATS cases unless on snapshot (#111755) Fix https://github.com/elastic/elasticsearch/issues/111753 --- muted-tests.yml | 3 - .../xpack/esql/analysis/VerifierTests.java | 62 ++++++++++--------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 1fc04d84e3173..e439759d75dab 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -125,9 +125,6 @@ tests: - class: org.elasticsearch.tdigest.ComparisonTests method: testSparseGaussianDistribution issue: https://github.com/elastic/elasticsearch/issues/111721 -- class: org.elasticsearch.xpack.esql.analysis.VerifierTests - method: testUnsupportedAndMultiTypedFields - issue: https://github.com/elastic/elasticsearch/issues/111753 # Examples: # diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 6e9cda60f7d5d..904308ef64d58 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -138,15 +138,17 @@ public void testUnsupportedAndMultiTypedFields() { + " [ip] in [test1, test2], [keyword] in [test3]", error("from test* | stats count(1) by multi_typed", analyzer) ); - assertEquals( - "1:38: Cannot use field [unsupported] with unsupported type [flattened]", - error("from test* | inlinestats count(1) by unsupported", analyzer) - ); - assertEquals( - "1:38: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" - + " [ip] in [test1, test2], [keyword] in [test3]", - error("from test* | inlinestats count(1) by multi_typed", analyzer) - ); + if (EsqlCapabilities.Cap.INLINESTATS.isEnabled()) { + assertEquals( + "1:38: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | inlinestats count(1) by unsupported", analyzer) + ); + assertEquals( + "1:38: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | inlinestats count(1) by multi_typed", analyzer) + ); + } assertEquals( "1:27: Cannot use field [unsupported] with unsupported type [flattened]", @@ -157,15 +159,17 @@ public void testUnsupportedAndMultiTypedFields() { + " [ip] in [test1, test2], [keyword] in [test3]", error("from test* | stats values(multi_typed)", analyzer) ); - assertEquals( - "1:33: Cannot use field [unsupported] with unsupported type [flattened]", - error("from test* | inlinestats values(unsupported)", analyzer) - ); - assertEquals( - "1:33: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" - + " [ip] in [test1, test2], [keyword] in [test3]", - error("from test* | inlinestats values(multi_typed)", analyzer) - ); + if (EsqlCapabilities.Cap.INLINESTATS.isEnabled()) { + assertEquals( + "1:33: Cannot use field [unsupported] with unsupported type [flattened]", + error("from test* | inlinestats values(unsupported)", analyzer) + ); + assertEquals( + "1:33: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:" + + " [ip] in [test1, test2], [keyword] in [test3]", + error("from test* | inlinestats values(multi_typed)", analyzer) + ); + } assertEquals( "1:27: Cannot use field [unsupported] with unsupported type [flattened]", @@ -177,16 +181,18 @@ public void testUnsupportedAndMultiTypedFields() { error("from test* | stats values(multi_typed)", analyzer) ); - // LOOKUP with unsupported type - assertEquals( - "1:41: column type mismatch, table column was [integer] and original column was [unsupported]", - error("from test* | lookup int_number_names on int", analyzer) - ); - // LOOKUP with multi-typed field - assertEquals( - "1:44: column type mismatch, table column was [double] and original column was [unsupported]", - error("from test* | lookup double_number_names on double", analyzer) - ); + if (EsqlCapabilities.Cap.LOOKUP_V4.isEnabled()) { + // LOOKUP with unsupported type + assertEquals( + "1:41: column type mismatch, table column was [integer] and original column was [unsupported]", + error("from test* | lookup int_number_names on int", analyzer) + ); + // LOOKUP with multi-typed field + assertEquals( + "1:44: column type mismatch, table column was [double] and original column was [unsupported]", + error("from test* | lookup double_number_names on double", analyzer) + ); + } assertEquals( "1:24: Cannot use field [unsupported] with unsupported type [flattened]", From 13cc380a848f0102500070a2ac3ac9e7b435c56a Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Fri, 9 Aug 2024 17:22:59 +0200 Subject: [PATCH 24/37] EIS integration (#111154) * WIP * Add ElasticInferenceServiceTests TODOs * Add ElasticInferenceServiceActionCreatorTests TODOs * Add ElasticInferenceServiceResponseHandlerTests TODOs * Add ElasticInferenceServiceSparseEmbeddingsRequestTests TODOs * Add ElasticInferenceServiceSparseEmbeddingsModelTests TODOs * spotless apply * Fix conflicts * Add EmptySecretSettingsTests * Add named writeables to InferenceNamedWriteablesProvider * Remove addressed todos * Translate model to correct endpoint * Remove addressed TODO * Add docs to ElasticInferenceServiceFeature * Implement and test truncation/request * Add some EIS tests * Support chunked inference * Check model config * Add more tests * Add response handler * Add more tests + HTTP 413 handling * Fix some tests * Spotless * Fixes * Switch back to original response structure * Implement pass-through chunking * Spotless * Fix after rebase * Spotless * Log error upon failing to parse error response * Remove TODOs * Update docs/changelog/111154.yaml --------- Co-authored-by: Adam Demjen --- docs/changelog/111154.yaml | 5 + .../org/elasticsearch/TransportVersions.java | 1 + .../inference/EmptySecretSettings.java | 50 ++ .../InferenceNamedWriteablesProvider.java | 16 + .../xpack/inference/InferencePlugin.java | 16 +- .../ElasticInferenceServiceActionCreator.java | 38 ++ .../ElasticInferenceServiceActionVisitor.java | 17 + ...lasticInferenceServiceResponseHandler.java | 54 ++ .../http/retry/BaseResponseHandler.java | 2 + ...ElasticInferenceServiceRequestManager.java | 28 + ...ServiceSparseEmbeddingsRequestManager.java | 71 +++ .../ElasticInferenceServiceRequest.java | 12 + ...ferenceServiceSparseEmbeddingsRequest.java | 80 +++ ...eServiceSparseEmbeddingsRequestEntity.java | 41 ++ ...icInferenceServiceErrorResponseEntity.java | 64 +++ ...ServiceSparseEmbeddingsResponseEntity.java | 121 ++++ .../elastic/ElasticInferenceService.java | 274 +++++++++ .../ElasticInferenceServiceComponents.java | 10 + .../ElasticInferenceServiceFeature.java | 20 + .../elastic/ElasticInferenceServiceModel.java | 55 ++ ...erenceServiceRateLimitServiceSettings.java | 18 + .../ElasticInferenceServiceSettings.java | 33 ++ ...InferenceServiceSparseEmbeddingsModel.java | 113 ++++ ...erviceSparseEmbeddingsServiceSettings.java | 162 ++++++ .../services/elser/ElserInternalService.java | 13 +- .../elser/ElserInternalServiceSettings.java | 6 +- .../inference/services/elser/ElserModels.java | 33 ++ .../inference/EmptySecretSettingsTests.java | 35 ++ ...ticInferenceServiceActionCreatorTests.java | 289 ++++++++++ ...cInferenceServiceResponseHandlerTests.java | 116 ++++ ...iceSparseEmbeddingsRequestEntityTests.java | 50 ++ ...ceServiceSparseEmbeddingsRequestTests.java | 81 +++ ...erenceServiceErrorResponseEntityTests.java | 61 ++ ...ceSparseEmbeddingsResponseEntityTests.java | 241 ++++++++ ...enceServiceSparseEmbeddingsModelTests.java | 33 ++ ...eSparseEmbeddingsServiceSettingsTests.java | 90 +++ .../elastic/ElasticInferenceServiceTests.java | 523 ++++++++++++++++++ .../ElserInternalServiceSettingsTests.java | 8 +- .../elser/ElserInternalServiceTests.java | 2 +- .../services/elser/ElserModelsTests.java | 33 ++ 40 files changed, 2894 insertions(+), 21 deletions(-) create mode 100644 docs/changelog/111154.yaml create mode 100644 server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java diff --git a/docs/changelog/111154.yaml b/docs/changelog/111154.yaml new file mode 100644 index 0000000000000..3297f5005a811 --- /dev/null +++ b/docs/changelog/111154.yaml @@ -0,0 +1,5 @@ +pr: 111154 +summary: EIS integration +area: Inference +type: feature +issues: [] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 2427a2fe72ac6..2b579894ca521 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -187,6 +187,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_NESTED_UNSUPPORTED = def(8_717_00_0); public static final TransportVersion ESQL_SINGLE_VALUE_QUERY_SOURCE = def(8_718_00_0); public static final TransportVersion ESQL_ORIGINAL_INDICES = def(8_719_00_0); + public static final TransportVersion ML_INFERENCE_EIS_INTEGRATION_ADDED = def(8_720_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java new file mode 100644 index 0000000000000..5c6acb78a91e3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.inference; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * This class defines an empty secret settings object. This is useful for services that do not have any secret settings. + */ +public record EmptySecretSettings() implements SecretSettings { + public static final String NAME = "empty_secret_settings"; + + public static final EmptySecretSettings INSTANCE = new EmptySecretSettings(); + + public EmptySecretSettings(StreamInput in) { + this(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 476ab3355a0b8..489a81b642492 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.inference.EmptySecretSettings; import org.elasticsearch.inference.EmptyTaskSettings; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; @@ -45,6 +46,7 @@ import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsTaskSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings; @@ -95,6 +97,9 @@ public static List getNamedWriteables() { // Empty default task settings namedWriteables.add(new NamedWriteableRegistry.Entry(TaskSettings.class, EmptyTaskSettings.NAME, EmptyTaskSettings::new)); + // Empty default secret settings + namedWriteables.add(new NamedWriteableRegistry.Entry(SecretSettings.class, EmptySecretSettings.NAME, EmptySecretSettings::new)); + // Default secret settings namedWriteables.add(new NamedWriteableRegistry.Entry(SecretSettings.class, DefaultSecretSettings.NAME, DefaultSecretSettings::new)); @@ -111,6 +116,7 @@ public static List getNamedWriteables() { addCustomElandWriteables(namedWriteables); addAnthropicNamedWritables(namedWriteables); addAmazonBedrockNamedWriteables(namedWriteables); + addEisNamedWriteables(namedWriteables); return namedWriteables; } @@ -475,4 +481,14 @@ private static void addAnthropicNamedWritables(List namedWriteables) { + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + ElasticInferenceServiceSparseEmbeddingsServiceSettings.NAME, + ElasticInferenceServiceSparseEmbeddingsServiceSettings::new + ) + ); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index ec9398358d180..f6d4a9f774a91 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -77,6 +77,10 @@ import org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioService; import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiService; import org.elasticsearch.xpack.inference.services.cohere.CohereService; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceComponents; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceFeature; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; import org.elasticsearch.xpack.inference.services.elser.ElserInternalService; import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService; @@ -124,7 +128,7 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP private final SetOnce httpFactory = new SetOnce<>(); private final SetOnce amazonBedrockFactory = new SetOnce<>(); private final SetOnce serviceComponents = new SetOnce<>(); - + private final SetOnce eisComponents = new SetOnce<>(); private final SetOnce inferenceServiceRegistry = new SetOnce<>(); private final SetOnce shardBulkInferenceActionFilter = new SetOnce<>(); private List inferenceServiceExtensions; @@ -187,6 +191,15 @@ public Collection createComponents(PluginServices services) { var inferenceServices = new ArrayList<>(inferenceServiceExtensions); inferenceServices.add(this::getInferenceServiceFactories); + if (ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()) { + ElasticInferenceServiceSettings eisSettings = new ElasticInferenceServiceSettings(settings); + eisComponents.set(new ElasticInferenceServiceComponents(eisSettings.getEisGatewayUrl())); + + inferenceServices.add( + () -> List.of(context -> new ElasticInferenceService(httpFactory.get(), serviceComponents.get(), eisComponents.get())) + ); + } + var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext(services.client()); // This must be done after the HttpRequestSenderFactory is created so that the services can get the // reference correctly @@ -281,6 +294,7 @@ public List> getSettings() { HttpClientManager.getSettingsDefinitions(), ThrottlerManager.getSettingsDefinitions(), RetrySettings.getSettingsDefinitions(), + ElasticInferenceServiceSettings.getSettingsDefinitions(), Truncator.getSettingsDefinitions(), RequestExecutorServiceSettings.getSettingsDefinitions(), List.of(SKIP_VALIDATE_AND_START) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java new file mode 100644 index 0000000000000..ea2295979c480 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.elastic; + +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.ElasticInferenceServiceSparseEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel; + +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + +public class ElasticInferenceServiceActionCreator implements ElasticInferenceServiceActionVisitor { + + private final Sender sender; + + private final ServiceComponents serviceComponents; + + public ElasticInferenceServiceActionCreator(Sender sender, ServiceComponents serviceComponents) { + this.sender = Objects.requireNonNull(sender); + this.serviceComponents = Objects.requireNonNull(serviceComponents); + } + + @Override + public ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model) { + var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents); + var errorMessage = constructFailedToSendRequestMessage(model.uri(), "Elastic Inference Service sparse embeddings"); + return new SenderExecutableAction(sender, requestManager, errorMessage); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java new file mode 100644 index 0000000000000..99985e50b2538 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.elastic; + +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel; + +public interface ElasticInferenceServiceActionVisitor { + + ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java new file mode 100644 index 0000000000000..15e543fadad71 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.elastic; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; +import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceErrorResponseEntity; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; + +import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; + +public class ElasticInferenceServiceResponseHandler extends BaseResponseHandler { + + public ElasticInferenceServiceResponseHandler(String requestType, ResponseParser parseFunction) { + super(requestType, parseFunction, ElasticInferenceServiceErrorResponseEntity::fromResponse); + } + + @Override + public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) + throws RetryException { + checkForFailureStatusCode(request, result); + checkForEmptyBody(throttlerManager, logger, request, result); + } + + void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + int statusCode = result.response().getStatusLine().getStatusCode(); + if (statusCode >= 200 && statusCode < 300) { + return; + } + + if (statusCode == 500) { + throw new RetryException(true, buildError(SERVER_ERROR, request, result)); + } else if (statusCode == 400) { + throw new RetryException(false, buildError(BAD_REQUEST, request, result)); + } else if (statusCode == 405) { + throw new RetryException(false, buildError(METHOD_NOT_ALLOWED, request, result)); + } else if (statusCode == 413) { + throw new ContentTooLargeException(buildError(CONTENT_TOO_LARGE, request, result)); + } + + throw new RetryException(false, buildError(UNSUCCESSFUL, request, result)); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java index f793cb3586924..c9cbe169ec03d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java @@ -27,6 +27,8 @@ public abstract class BaseResponseHandler implements ResponseHandler { public static final String REDIRECTION = "Unhandled redirection"; public static final String CONTENT_TOO_LARGE = "Received a content too large status code"; public static final String UNSUCCESSFUL = "Received an unsuccessful status code"; + public static final String BAD_REQUEST = "Received a bad request status code"; + public static final String METHOD_NOT_ALLOWED = "Received a method not allowed status code"; protected final String requestType; private final ResponseParser parseFunction; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java new file mode 100644 index 0000000000000..c857a481f8f04 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceModel; + +import java.util.Objects; + +public abstract class ElasticInferenceServiceRequestManager extends BaseRequestManager { + + protected ElasticInferenceServiceRequestManager(ThreadPool threadPool, ElasticInferenceServiceModel model) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitServiceSettings().rateLimitSettings()); + } + + record RateLimitGrouping(int modelIdHash) { + public static RateLimitGrouping of(ElasticInferenceServiceModel model) { + Objects.requireNonNull(model); + + return new RateLimitGrouping(model.rateLimitServiceSettings().modelId().hashCode()); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java new file mode 100644 index 0000000000000..b59ac54d5cbb6 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.inference.common.Truncator; +import org.elasticsearch.xpack.inference.external.elastic.ElasticInferenceServiceResponseHandler; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.request.elastic.ElasticInferenceServiceSparseEmbeddingsRequest; +import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceSparseEmbeddingsResponseEntity; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel; + +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.inference.common.Truncator.truncate; + +public class ElasticInferenceServiceSparseEmbeddingsRequestManager extends ElasticInferenceServiceRequestManager { + + private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceSparseEmbeddingsRequestManager.class); + + private static final ResponseHandler HANDLER = createSparseEmbeddingsHandler(); + + private final ElasticInferenceServiceSparseEmbeddingsModel model; + + private final Truncator truncator; + + private static ResponseHandler createSparseEmbeddingsHandler() { + return new ElasticInferenceServiceResponseHandler( + "Elastic Inference Service sparse embeddings", + ElasticInferenceServiceSparseEmbeddingsResponseEntity::fromResponse + ); + } + + public ElasticInferenceServiceSparseEmbeddingsRequestManager( + ElasticInferenceServiceSparseEmbeddingsModel model, + ServiceComponents serviceComponents + ) { + super(serviceComponents.threadPool(), model); + this.model = model; + this.truncator = serviceComponents.truncator(); + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); + var truncatedInput = truncate(docsInput, model.getServiceSettings().maxInputTokens()); + + ElasticInferenceServiceSparseEmbeddingsRequest request = new ElasticInferenceServiceSparseEmbeddingsRequest( + truncator, + truncatedInput, + model + ); + execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java new file mode 100644 index 0000000000000..03eec913a265f --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.elastic; + +import org.elasticsearch.xpack.inference.external.request.Request; + +public interface ElasticInferenceServiceRequest extends Request {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java new file mode 100644 index 0000000000000..41a2ef1c3ccda --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.elastic; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.common.Truncator; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +public class ElasticInferenceServiceSparseEmbeddingsRequest implements ElasticInferenceServiceRequest { + + private final URI uri; + + private final ElasticInferenceServiceSparseEmbeddingsModel model; + + private final Truncator.TruncationResult truncationResult; + private final Truncator truncator; + + public ElasticInferenceServiceSparseEmbeddingsRequest( + Truncator truncator, + Truncator.TruncationResult truncationResult, + ElasticInferenceServiceSparseEmbeddingsModel model + ) { + this.truncator = truncator; + this.truncationResult = truncationResult; + this.model = Objects.requireNonNull(model); + this.uri = model.uri(); + } + + @Override + public HttpRequest createHttpRequest() { + var httpPost = new HttpPost(uri); + var requestEntity = Strings.toString(new ElasticInferenceServiceSparseEmbeddingsRequestEntity(truncationResult.input())); + + ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8)); + httpPost.setEntity(byteEntity); + + httpPost.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType())); + + return new HttpRequest(httpPost, getInferenceEntityId()); + } + + @Override + public String getInferenceEntityId() { + return model.getInferenceEntityId(); + } + + @Override + public URI getURI() { + return this.uri; + } + + @Override + public Request truncate() { + var truncatedInput = truncator.truncate(truncationResult.input()); + + return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model); + } + + @Override + public boolean[] getTruncationInfo() { + return truncationResult.truncated().clone(); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java new file mode 100644 index 0000000000000..301bbf0146c14 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.elastic; + +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public record ElasticInferenceServiceSparseEmbeddingsRequestEntity(List inputs) implements ToXContentObject { + + private static final String INPUT_FIELD = "input"; + + public ElasticInferenceServiceSparseEmbeddingsRequestEntity { + Objects.requireNonNull(inputs); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startArray(INPUT_FIELD); + + { + for (String input : inputs) { + builder.value(input); + } + } + + builder.endArray(); + builder.endObject(); + + return builder; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java new file mode 100644 index 0000000000000..c860821c81bbf --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.elastic; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; + +public class ElasticInferenceServiceErrorResponseEntity implements ErrorMessage { + + private final String errorMessage; + + private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceErrorResponseEntity.class); + + private ElasticInferenceServiceErrorResponseEntity(String errorMessage) { + this.errorMessage = errorMessage; + } + + @Override + public String getErrorMessage() { + return errorMessage; + } + + /** + * An example error response would look like + * + * + * { + * "error": "some error" + * } + * + * + * @param response The error response + * @return An error entity if the response is JSON with the above structure + * or null if the response does not contain the error field + */ + public static @Nullable ElasticInferenceServiceErrorResponseEntity fromResponse(HttpResult response) { + try ( + XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, response.body()) + ) { + var responseMap = jsonParser.map(); + var error = (String) responseMap.get("error"); + if (error != null) { + return new ElasticInferenceServiceErrorResponseEntity(error); + } + } catch (Exception e) { + logger.debug("Failed to parse error response", e); + } + + return null; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java new file mode 100644 index 0000000000000..2b36cc5d22cd4 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.elastic; + +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.search.WeightedToken; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.Request; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField; + +public class ElasticInferenceServiceSparseEmbeddingsResponseEntity { + + private static final String FAILED_TO_FIND_FIELD_TEMPLATE = + "Failed to find required field [%s] in Elastic Inference Service embeddings response"; + + /** + * Parses the EIS json response. + * + * For a request like: + * + * 

+     *     
+     *         {
+     *             "inputs": ["Embed this text", "Embed this text, too"]
+     *         }
+     *     
+     *
+ * + * The response would look like: + * + * 
+     *     
+     *         {
+     *           "data": [
+     *                     {
+     *                       "Embed": 2.1259406,
+     *                       "this": 1.7073475,
+     *                       "text": 0.9020516
+     *                     },
+     *                    (...)
+     *                  ],
+     *           "meta": {
+     *               "processing_latency": ...,
+     *               "request_time": ...
+     *           }
+     *     
+     *
+ */ + + public static SparseEmbeddingResults fromResponse(Request request, HttpResult response) throws IOException { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { + moveToFirstToken(jsonParser); + + XContentParser.Token token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + + positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE); + + var truncationResults = request.getTruncationInfo(); + List parsedEmbeddings = parseList( + jsonParser, + (parser, index) -> ElasticInferenceServiceSparseEmbeddingsResponseEntity.parseExpansionResult( + truncationResults, + parser, + index + ) + ); + + if (parsedEmbeddings.isEmpty()) { + return new SparseEmbeddingResults(Collections.emptyList()); + } + + return new SparseEmbeddingResults(parsedEmbeddings); + } + } + + private static SparseEmbeddingResults.Embedding parseExpansionResult(boolean[] truncationResults, XContentParser parser, int index) + throws IOException { + XContentParser.Token token = parser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser); + + List weightedTokens = new ArrayList<>(); + token = parser.nextToken(); + while (token != null && token != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser); + var floatToken = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, floatToken, parser); + + weightedTokens.add(new WeightedToken(parser.currentName(), parser.floatValue())); + + token = parser.nextToken(); + } + + // prevent an out of bounds if for some reason the truncation list is smaller than the results + var isTruncated = truncationResults != null && index < truncationResults.length && truncationResults[index]; + return new SparseEmbeddingResults.Embedding(weightedTokens, isTruncated); + } + + private ElasticInferenceServiceSparseEmbeddingsResponseEntity() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java new file mode 100644 index 0000000000000..f77217f9c02f9 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -0,0 +1,274 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; +import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionCreator; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.SenderService; +import org.elasticsearch.xpack.inference.services.ServiceComponents; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; + +public class ElasticInferenceService extends SenderService { + + public static final String NAME = "elastic"; + + private final ElasticInferenceServiceComponents elasticInferenceServiceComponents; + + public ElasticInferenceService( + HttpRequestSender.Factory factory, + ServiceComponents serviceComponents, + ElasticInferenceServiceComponents eisComponents + ) { + super(factory, serviceComponents); + this.elasticInferenceServiceComponents = eisComponents; + } + + @Override + protected void doInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + TimeValue timeout, + ActionListener listener + ) { + if (model instanceof ElasticInferenceServiceModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + ElasticInferenceServiceModel elasticInferenceServiceModel = (ElasticInferenceServiceModel) model; + var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents()); + + var action = elasticInferenceServiceModel.accept(actionCreator, taskSettings); + action.execute(new DocumentsOnlyInput(input), timeout, listener); + } + + @Override + protected void doInfer( + Model model, + String query, + List input, + Map taskSettings, + InputType inputType, + TimeValue timeout, + ActionListener listener + ) { + throw new UnsupportedOperationException("Query input not supported for Elastic Inference Service"); + } + + @Override + protected void doChunkedInfer( + Model model, + String query, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + TimeValue timeout, + ActionListener> listener + ) { + // Pass-through without actually performing chunking (result will have a single chunk per input) + ActionListener inferListener = listener.delegateFailureAndWrap( + (delegate, response) -> delegate.onResponse(translateToChunkedResults(input, response)) + ); + + doInfer(model, input, taskSettings, inputType, timeout, inferListener); + } + + @Override + public String name() { + return NAME; + } + + @Override + public void parseRequestConfig( + String inferenceEntityId, + TaskType taskType, + Map config, + Set platformArchitectures, + ActionListener parsedModelListener + ) { + try { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + ElasticInferenceServiceModel model = createModel( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + serviceSettingsMap, + elasticInferenceServiceComponents, + TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), + ConfigurationParseContext.REQUEST + ); + + throwIfNotEmptyMap(config, NAME); + throwIfNotEmptyMap(serviceSettingsMap, NAME); + throwIfNotEmptyMap(taskSettingsMap, NAME); + + parsedModelListener.onResponse(model); + } catch (Exception e) { + parsedModelListener.onFailure(e); + } + } + + private static ElasticInferenceServiceModel createModel( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + @Nullable Map secretSettings, + ElasticInferenceServiceComponents eisServiceComponents, + String failureMessage, + ConfigurationParseContext context + ) { + return switch (taskType) { + case SPARSE_EMBEDDING -> new ElasticInferenceServiceSparseEmbeddingsModel( + inferenceEntityId, + taskType, + NAME, + serviceSettings, + taskSettings, + secretSettings, + eisServiceComponents, + context + ); + default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST); + }; + } + + @Override + public Model parsePersistedConfigWithSecrets( + String inferenceEntityId, + TaskType taskType, + Map config, + Map secrets + ) { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); + Map secretSettingsMap = removeFromMapOrDefaultEmpty(secrets, ModelSecrets.SECRET_SETTINGS); + + return createModelFromPersistent( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + secretSettingsMap, + parsePersistedConfigErrorMsg(inferenceEntityId, NAME) + ); + } + + @Override + public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + return createModelFromPersistent( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + null, + parsePersistedConfigErrorMsg(inferenceEntityId, NAME) + ); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED; + } + + private ElasticInferenceServiceModel createModelFromPersistent( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + @Nullable Map secretSettings, + String failureMessage + ) { + return createModel( + inferenceEntityId, + taskType, + serviceSettings, + taskSettings, + secretSettings, + elasticInferenceServiceComponents, + failureMessage, + ConfigurationParseContext.PERSISTENT + ); + } + + @Override + public void checkModelConfig(Model model, ActionListener listener) { + if (model instanceof ElasticInferenceServiceSparseEmbeddingsModel embeddingsModel) { + listener.onResponse(updateModelWithEmbeddingDetails(embeddingsModel)); + } else { + listener.onResponse(model); + } + } + + private static List translateToChunkedResults( + List inputs, + InferenceServiceResults inferenceResults + ) { + if (inferenceResults instanceof SparseEmbeddingResults sparseEmbeddingResults) { + return InferenceChunkedSparseEmbeddingResults.listOf(inputs, sparseEmbeddingResults); + } else if (inferenceResults instanceof ErrorInferenceResults error) { + return List.of(new ErrorChunkedInferenceResults(error.getException())); + } else { + String expectedClass = Strings.format("%s", SparseEmbeddingResults.class.getSimpleName()); + throw createInvalidChunkedResultException(expectedClass, inferenceResults.getWriteableName()); + } + } + + private ElasticInferenceServiceSparseEmbeddingsModel updateModelWithEmbeddingDetails( + ElasticInferenceServiceSparseEmbeddingsModel model + ) { + ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings = new ElasticInferenceServiceSparseEmbeddingsServiceSettings( + model.getServiceSettings().modelId(), + model.getServiceSettings().maxInputTokens(), + model.getServiceSettings().rateLimitSettings() + ); + + return new ElasticInferenceServiceSparseEmbeddingsModel(model, serviceSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java new file mode 100644 index 0000000000000..4386964e927d2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +public record ElasticInferenceServiceComponents(String eisGatewayUrl) {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java new file mode 100644 index 0000000000000..b0fb6d14ee6f7 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.common.util.FeatureFlag; + +/** + * Elastic Inference Service (EIS) feature flag. When the feature is complete, this flag will be removed. + * Enable feature via JVM option: `-Des.eis_feature_flag_enabled=true`. + */ +public class ElasticInferenceServiceFeature { + + public static final FeatureFlag ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG = new FeatureFlag("eis"); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java new file mode 100644 index 0000000000000..e7809d869fec4 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor; + +import java.util.Map; +import java.util.Objects; + +public abstract class ElasticInferenceServiceModel extends Model { + + private final ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings; + + private final ElasticInferenceServiceComponents elasticInferenceServiceComponents; + + public ElasticInferenceServiceModel( + ModelConfigurations configurations, + ModelSecrets secrets, + ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings, + ElasticInferenceServiceComponents elasticInferenceServiceComponents + ) { + super(configurations, secrets); + + this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings); + this.elasticInferenceServiceComponents = Objects.requireNonNull(elasticInferenceServiceComponents); + } + + public ElasticInferenceServiceModel(ElasticInferenceServiceModel model, ServiceSettings serviceSettings) { + super(model, serviceSettings); + + this.rateLimitServiceSettings = model.rateLimitServiceSettings(); + this.elasticInferenceServiceComponents = model.elasticInferenceServiceComponents(); + } + + public ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings() { + return rateLimitServiceSettings; + } + + public ElasticInferenceServiceComponents elasticInferenceServiceComponents() { + return elasticInferenceServiceComponents; + } + + public abstract ExecutableAction accept(ElasticInferenceServiceActionVisitor visitor, Map taskSettings); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java new file mode 100644 index 0000000000000..2ec562b61fa01 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +public interface ElasticInferenceServiceRateLimitServiceSettings { + + String modelId(); + + RateLimitSettings rateLimitSettings(); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java new file mode 100644 index 0000000000000..8525710c6cf23 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; + +import java.util.List; + +public class ElasticInferenceServiceSettings { + + static final Setting EIS_GATEWAY_URL = Setting.simpleString("xpack.inference.eis.gateway.url", Setting.Property.NodeScope); + + // Adjust this variable to be volatile, if the setting can be updated at some point in time + private final String eisGatewayUrl; + + public ElasticInferenceServiceSettings(Settings settings) { + eisGatewayUrl = EIS_GATEWAY_URL.get(settings); + } + + public static List> getSettingsDefinitions() { + return List.of(EIS_GATEWAY_URL); + } + + public String getEisGatewayUrl() { + return eisGatewayUrl; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java new file mode 100644 index 0000000000000..163e3dd654150 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.EmptySecretSettings; +import org.elasticsearch.inference.EmptyTaskSettings; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.SecretSettings; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.elser.ElserModels; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +public class ElasticInferenceServiceSparseEmbeddingsModel extends ElasticInferenceServiceModel { + + private final URI uri; + + public ElasticInferenceServiceSparseEmbeddingsModel( + String inferenceEntityId, + TaskType taskType, + String service, + Map serviceSettings, + Map taskSettings, + Map secrets, + ElasticInferenceServiceComponents elasticInferenceServiceComponents, + ConfigurationParseContext context + ) { + this( + inferenceEntityId, + taskType, + service, + ElasticInferenceServiceSparseEmbeddingsServiceSettings.fromMap(serviceSettings, context), + EmptyTaskSettings.INSTANCE, + EmptySecretSettings.INSTANCE, + elasticInferenceServiceComponents + ); + } + + public ElasticInferenceServiceSparseEmbeddingsModel( + ElasticInferenceServiceSparseEmbeddingsModel model, + ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings + ) { + super(model, serviceSettings); + + try { + this.uri = createUri(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + ElasticInferenceServiceSparseEmbeddingsModel( + String inferenceEntityId, + TaskType taskType, + String service, + ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings, + @Nullable TaskSettings taskSettings, + @Nullable SecretSettings secretSettings, + ElasticInferenceServiceComponents elasticInferenceServiceComponents + ) { + super( + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secretSettings), + serviceSettings, + elasticInferenceServiceComponents + ); + + try { + this.uri = createUri(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + public ExecutableAction accept(ElasticInferenceServiceActionVisitor visitor, Map taskSettings) { + return visitor.create(this); + } + + @Override + public ElasticInferenceServiceSparseEmbeddingsServiceSettings getServiceSettings() { + return (ElasticInferenceServiceSparseEmbeddingsServiceSettings) super.getServiceSettings(); + } + + public URI uri() { + return uri; + } + + private URI createUri() throws URISyntaxException { + String modelId = getServiceSettings().modelId(); + String modelIdUriPath; + + switch (modelId) { + case ElserModels.ELSER_V2_MODEL -> modelIdUriPath = "ELSERv2"; + default -> throw new IllegalArgumentException("Unsupported model for EIS [" + modelId + "]"); + } + + return new URI(elasticInferenceServiceComponents().eisGatewayUrl() + "/sparse-text-embedding/" + modelIdUriPath); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java new file mode 100644 index 0000000000000..15b89525f7915 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.elser.ElserModels; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; + +public class ElasticInferenceServiceSparseEmbeddingsServiceSettings extends FilteredXContentObject + implements + ServiceSettings, + ElasticInferenceServiceRateLimitServiceSettings { + + public static final String NAME = "elastic_inference_service_sparse_embeddings_service_settings"; + + private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(1_000); + + public static ElasticInferenceServiceSparseEmbeddingsServiceSettings fromMap( + Map map, + ConfigurationParseContext context + ) { + ValidationException validationException = new ValidationException(); + + String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + Integer maxInputTokens = extractOptionalPositiveInteger( + map, + MAX_INPUT_TOKENS, + ModelConfigurations.SERVICE_SETTINGS, + validationException + ); + + RateLimitSettings rateLimitSettings = RateLimitSettings.of( + map, + DEFAULT_RATE_LIMIT_SETTINGS, + validationException, + ElasticInferenceService.NAME, + context + ); + + if (modelId != null && ElserModels.isValidEisModel(modelId) == false) { + validationException.addValidationError("unknown ELSER model id [" + modelId + "]"); + } + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, maxInputTokens, rateLimitSettings); + } + + private final String modelId; + + private final Integer maxInputTokens; + private final RateLimitSettings rateLimitSettings; + + public ElasticInferenceServiceSparseEmbeddingsServiceSettings( + String modelId, + @Nullable Integer maxInputTokens, + RateLimitSettings rateLimitSettings + ) { + this.modelId = Objects.requireNonNull(modelId); + this.maxInputTokens = maxInputTokens; + this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS); + } + + public ElasticInferenceServiceSparseEmbeddingsServiceSettings(StreamInput in) throws IOException { + this.modelId = in.readString(); + this.maxInputTokens = in.readOptionalVInt(); + this.rateLimitSettings = new RateLimitSettings(in); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public String modelId() { + return modelId; + } + + public Integer maxInputTokens() { + return maxInputTokens; + } + + @Override + public RateLimitSettings rateLimitSettings() { + return rateLimitSettings; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + toXContentFragmentOfExposedFields(builder, params); + + builder.endObject(); + + return builder; + } + + @Override + protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + builder.field(MODEL_ID, modelId); + if (maxInputTokens != null) { + builder.field(MAX_INPUT_TOKENS, maxInputTokens); + } + rateLimitSettings.toXContent(builder, params); + + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(modelId); + out.writeOptionalVInt(maxInputTokens); + rateLimitSettings.writeTo(out); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + ElasticInferenceServiceSparseEmbeddingsServiceSettings that = (ElasticInferenceServiceSparseEmbeddingsServiceSettings) object; + return Objects.equals(modelId, that.modelId) + && Objects.equals(maxInputTokens, that.maxInputTokens) + && Objects.equals(rateLimitSettings, that.rateLimitSettings); + } + + @Override + public int hashCode() { + return Objects.hash(modelId, maxInputTokens, rateLimitSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index 03d7682600e7c..775ddca160463 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -47,22 +47,13 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.elser.ElserModels.ELSER_V2_MODEL; +import static org.elasticsearch.xpack.inference.services.elser.ElserModels.ELSER_V2_MODEL_LINUX_X86; public class ElserInternalService extends BaseElasticsearchInternalService { public static final String NAME = "elser"; - static final String ELSER_V1_MODEL = ".elser_model_1"; - // Default non platform specific v2 model - static final String ELSER_V2_MODEL = ".elser_model_2"; - static final String ELSER_V2_MODEL_LINUX_X86 = ".elser_model_2_linux-x86_64"; - - public static Set VALID_ELSER_MODEL_IDS = Set.of( - ElserInternalService.ELSER_V1_MODEL, - ElserInternalService.ELSER_V2_MODEL, - ElserInternalService.ELSER_V2_MODEL_LINUX_X86 - ); - private static final String OLD_MODEL_ID_FIELD_NAME = "model_version"; public ElserInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java index 75797919b3616..fcbabd5a88fc6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java @@ -18,8 +18,6 @@ import java.util.Arrays; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.elser.ElserInternalService.VALID_ELSER_MODEL_IDS; - public class ElserInternalServiceSettings extends ElasticsearchInternalServiceSettings { public static final String NAME = "elser_mlnode_service_settings"; @@ -29,10 +27,10 @@ public static ElasticsearchInternalServiceSettings.Builder fromRequestMap(Map VALID_ELSER_MODEL_IDS = Set.of( + ElserModels.ELSER_V1_MODEL, + ElserModels.ELSER_V2_MODEL, + ElserModels.ELSER_V2_MODEL_LINUX_X86 + ); + + public static boolean isValidModel(String model) { + return VALID_ELSER_MODEL_IDS.contains(model); + } + + public static boolean isValidEisModel(String model) { + return ELSER_V2_MODEL.equals(model); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java new file mode 100644 index 0000000000000..b50ea9e5ee224 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.inference.EmptySecretSettings; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +public class EmptySecretSettingsTests extends AbstractWireSerializingTestCase { + + public static EmptySecretSettings createRandom() { + return EmptySecretSettings.INSTANCE; // no options to randomise + } + + @Override + protected Writeable.Reader instanceReader() { + return EmptySecretSettings::new; + } + + @Override + protected EmptySecretSettings createTestInstance() { + return createRandom(); + } + + @Override + protected EmptySecretSettings mutateInstance(EmptySecretSettings instance) { + // All instances are the same and have no fields, nothing to mutate + return null; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java new file mode 100644 index 0000000000000..1081a60ba6866 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java @@ -0,0 +1,289 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.elastic; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.buildSettingsWithRetryFields; +import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class ElasticInferenceServiceActionCreatorTests extends ESTestCase { + + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + @SuppressWarnings("unchecked") + public void testExecute_ReturnsSuccessfulResponse_ForElserAction() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = createSender(senderFactory)) { + sender.start(); + + String responseJson = """ + { + "data": [ + { + "hello": 2.1259406, + "greet": 1.7073475 + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer)); + var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool)); + var action = actionCreator.create(model); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertThat( + result.asMap(), + is( + SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings( + List.of( + new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), false) + ) + ) + ) + ); + + assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + assertThat(requestMap.size(), is(1)); + assertThat(requestMap.get("input"), instanceOf(List.class)); + var inputList = (List) requestMap.get("input"); + assertThat(inputList, contains("hello world")); + } + } + + @SuppressWarnings("unchecked") + public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOException { + // timeout as zero for no retries + var settings = buildSettingsWithRetryFields( + TimeValue.timeValueMillis(1), + TimeValue.timeValueMinutes(1), + TimeValue.timeValueSeconds(0) + ); + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, settings); + + try (var sender = createSender(senderFactory)) { + sender.start(); + + // This will fail because the expected output is {"data": [{...}]} + String responseJson = """ + { + "data": { + "hello": 2.1259406 + } + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer)); + var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool)); + var action = actionCreator.create(model); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + thrownException.getMessage(), + is("Failed to parse object: expecting token of type [START_ARRAY] but found [START_OBJECT]") + ); + + assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + assertThat(requestMap.size(), is(1)); + assertThat(requestMap.get("input"), instanceOf(List.class)); + var inputList = (List) requestMap.get("input"); + assertThat(inputList, contains("hello world")); + } + } + + public void testExecute_ReturnsSuccessfulResponse_AfterTruncating() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = createSender(senderFactory)) { + sender.start(); + + String responseJsonContentTooLarge = """ + { + "error": "Input validation error: `input` must have less than 512 tokens. Given: 571", + "error_type": "Validation" + } + """; + + String responseJson = """ + { + "data": [ + { + "hello": 2.1259406, + "greet": 1.7073475 + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(413).setBody(responseJsonContentTooLarge)); + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer)); + var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool)); + var action = actionCreator.create(model); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertThat( + result.asMap(), + is( + SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings( + List.of( + new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), true) + ) + ) + ) + ); + + assertThat(webServer.requests(), hasSize(2)); + { + assertNull(webServer.requests().get(0).getUri().getQuery()); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + + var initialRequestAsMap = entityAsMap(webServer.requests().get(0).getBody()); + var initialInputs = initialRequestAsMap.get("input"); + assertThat(initialInputs, is(List.of("hello world"))); + } + { + assertNull(webServer.requests().get(1).getUri().getQuery()); + assertThat(webServer.requests().get(1).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + + var truncatedRequest = entityAsMap(webServer.requests().get(1).getBody()); + var truncatedInputs = truncatedRequest.get("input"); + assertThat(truncatedInputs, is(List.of("hello"))); + } + } + } + + public void testExecute_TruncatesInputBeforeSending() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = createSender(senderFactory)) { + sender.start(); + + String responseJson = """ + { + "data": [ + { + "hello": 2.1259406, + "greet": 1.7073475 + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + // truncated to 1 token = 3 characters + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer), 1); + var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool)); + var action = actionCreator.create(model); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertThat( + result.asMap(), + is( + SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings( + List.of( + new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), true) + ) + ) + ) + ); + + assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + + var initialRequestAsMap = entityAsMap(webServer.requests().get(0).getBody()); + var initialInputs = initialRequestAsMap.get("input"); + assertThat(initialInputs, is(List.of("hel"))); + } + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java new file mode 100644 index 0000000000000..ea30ee29ff5a8 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.elastic; + +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.hamcrest.MatcherAssert; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.Is.is; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ElasticInferenceServiceResponseHandlerTests extends ESTestCase { + + public void testCheckForFailureStatusCode_DoesNotThrowFor200() { + callCheckForFailureStatusCode(200, "id"); + } + + public void testCheckForFailureStatusCode_ThrowsFor400() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(400, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a bad request status code for request from inference entity id [id] status [400]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor405() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(405, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a method not allowed status code for request from inference entity id [id] status [405]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.METHOD_NOT_ALLOWED)); + } + + public void testCheckForFailureStatusCode_ThrowsFor413() { + var exception = expectThrows(ContentTooLargeException.class, () -> callCheckForFailureStatusCode(413, "id")); + assertTrue(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a content too large status code for request from inference entity id [id] status [413]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.REQUEST_ENTITY_TOO_LARGE)); + } + + public void testCheckForFailureStatusCode_ThrowsFor500_WithShouldRetryTrue() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(500, "id")); + assertTrue(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a server error status code for request from inference entity id [id] status [500]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor402() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(402, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received an unsuccessful status code for request from inference entity id [id] status [402]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.PAYMENT_REQUIRED)); + } + + private static void callCheckForFailureStatusCode(int statusCode, String modelId) { + callCheckForFailureStatusCode(statusCode, null, modelId); + } + + private static void callCheckForFailureStatusCode(int statusCode, @Nullable String errorMessage, String modelId) { + var statusLine = mock(StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + + var httpResponse = mock(HttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + var header = mock(Header.class); + when(header.getElements()).thenReturn(new HeaderElement[] {}); + when(httpResponse.getFirstHeader(anyString())).thenReturn(header); + + String responseJson = Strings.format(""" + { + "message": "%s" + } + """, errorMessage); + + var mockRequest = mock(Request.class); + when(mockRequest.getInferenceEntityId()).thenReturn(modelId); + var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8)); + var handler = new ElasticInferenceServiceResponseHandler("", (request, result) -> null); + + handler.checkForFailureStatusCode(mockRequest, httpResult); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java new file mode 100644 index 0000000000000..7b10cf600275c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.elastic; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; + +public class ElasticInferenceServiceSparseEmbeddingsRequestEntityTests extends ESTestCase { + + public void testToXContent_SingleInput() throws IOException { + var entity = new ElasticInferenceServiceSparseEmbeddingsRequestEntity(List.of("abc")); + String xContentString = xContentEntityToString(entity); + assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" + { + "input": ["abc"] + }""")); + } + + public void testToXContent_MultipleInputs() throws IOException { + var entity = new ElasticInferenceServiceSparseEmbeddingsRequestEntity(List.of("abc", "def")); + String xContentString = xContentEntityToString(entity); + assertThat(xContentString, equalToIgnoringWhitespaceInJsonString(""" + { + "input": [ + "abc", + "def" + ] + } + """)); + } + + private String xContentEntityToString(ElasticInferenceServiceSparseEmbeddingsRequestEntity entity) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + return Strings.toString(builder); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java new file mode 100644 index 0000000000000..0f2c859fb62d5 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.elastic; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.common.Truncator; +import org.elasticsearch.xpack.inference.common.TruncatorTests; +import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class ElasticInferenceServiceSparseEmbeddingsRequestTests extends ESTestCase { + + public void testCreateHttpRequest() throws IOException { + var url = "http://eis-gateway.com"; + var input = "input"; + + var request = createRequest(url, input); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + assertThat(requestMap.size(), equalTo(1)); + assertThat(requestMap.get("input"), is(List.of(input))); + } + + public void testTruncate_ReducesInputTextSizeByHalf() throws IOException { + var url = "http://eis-gateway.com"; + var input = "abcd"; + + var request = createRequest(url, input); + var truncatedRequest = request.truncate(); + + var httpRequest = truncatedRequest.createHttpRequest(); + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + assertThat(requestMap, aMapWithSize(1)); + assertThat(requestMap.get("input"), is(List.of("ab"))); + } + + public void testIsTruncated_ReturnsTrue() { + var url = "http://eis-gateway.com"; + var input = "abcd"; + + var request = createRequest(url, input); + assertFalse(request.getTruncationInfo()[0]); + + var truncatedRequest = request.truncate(); + assertTrue(truncatedRequest.getTruncationInfo()[0]); + } + + public ElasticInferenceServiceSparseEmbeddingsRequest createRequest(String url, String input) { + var embeddingsModel = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(url); + + return new ElasticInferenceServiceSparseEmbeddingsRequest( + TruncatorTests.createTruncator(), + new Truncator.TruncationResult(List.of(input), new boolean[] { false }), + embeddingsModel + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java new file mode 100644 index 0000000000000..4da0518084828 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.elastic; + +import org.apache.http.HttpResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class ElasticInferenceServiceErrorResponseEntityTests extends ESTestCase { + + public void testFromResponse() { + String responseJson = """ + { + "error": "error" + } + """; + + ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertNotNull(errorResponseEntity); + assertThat(errorResponseEntity.getErrorMessage(), is("error")); + } + + public void testFromResponse_NoErrorMessagePresent() { + String responseJson = """ + { + "not_error": "error" + } + """; + + ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertNull(errorResponseEntity); + } + + public void testFromResponse_InvalidJson() { + String invalidResponseJson = """ + { + """; + + ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), invalidResponseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertNull(errorResponseEntity); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java new file mode 100644 index 0000000000000..6e1994260ca0c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.elastic; + +import org.apache.http.HttpResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.search.WeightedToken; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.Request; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ElasticInferenceServiceSparseEmbeddingsResponseEntityTests extends ESTestCase { + + public void testSparseEmbeddingsResponse_SingleEmbeddingInData_NoMeta_NoTruncation() throws Exception { + String responseJson = """ + { + "data": [ + { + "a": 1.23, + "is": 4.56, + "it": 7.89 + } + ] + } + """; + + SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)), + false + ) + ) + ) + ); + } + + public void testSparseEmbeddingsResponse_MultipleEmbeddingsInData_NoMeta_NoTruncation() throws Exception { + String responseJson = """ + { + "data": [ + { + "a": 1.23, + "is": 4.56, + "it": 7.89 + }, + { + "b": 1.23, + "it": 4.56, + "is": 7.89 + } + ] + } + """; + + SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)), + false + ), + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("b", 1.23F), new WeightedToken("it", 4.56F), new WeightedToken("is", 7.89F)), + false + ) + ) + ) + ); + } + + public void testSparseEmbeddingsResponse_SingleEmbeddingInData_NoMeta_Truncated() throws Exception { + String responseJson = """ + { + "data": [ + { + "a": 1.23, + "is": 4.56, + "it": 7.89 + } + ] + } + """; + + var request = mock(Request.class); + when(request.getTruncationInfo()).thenReturn(new boolean[] { true }); + + SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)), + true + ) + ) + ) + ); + } + + public void testSparseEmbeddingsResponse_MultipleEmbeddingsInData_NoMeta_Truncated() throws Exception { + String responseJson = """ + { + "data": [ + { + "a": 1.23, + "is": 4.56, + "it": 7.89 + }, + { + "b": 1.23, + "it": 4.56, + "is": 7.89 + } + ] + } + """; + + var request = mock(Request.class); + when(request.getTruncationInfo()).thenReturn(new boolean[] { true, false }); + + SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)), + true + ), + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("b", 1.23F), new WeightedToken("it", 4.56F), new WeightedToken("is", 7.89F)), + false + ) + ) + ) + ); + } + + public void testSparseEmbeddingsResponse_SingleEmbeddingInData_IgnoresMetaBeforeData_NoTruncation() throws Exception { + String responseJson = """ + { + "meta": { + "processing_latency": 1.23 + }, + "data": [ + { + "a": 1.23, + "is": 4.56, + "it": 7.89 + } + ] + } + """; + + SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)), + false + ) + ) + ) + ); + } + + public void testSparseEmbeddingsResponse_SingleEmbeddingInData_IgnoresMetaAfterData_NoTruncation() throws Exception { + String responseJson = """ + { + "data": [ + { + "a": 1.23, + "is": 4.56, + "it": 7.89 + } + ], + "meta": { + "processing_latency": 1.23 + } + } + """; + + SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + SparseEmbeddingResults.Embedding.create( + List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)), + false + ) + ) + ) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java new file mode 100644 index 0000000000000..af13ce7944685 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.inference.EmptySecretSettings; +import org.elasticsearch.inference.EmptyTaskSettings; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.elser.ElserModels; + +public class ElasticInferenceServiceSparseEmbeddingsModelTests extends ESTestCase { + + public static ElasticInferenceServiceSparseEmbeddingsModel createModel(String url) { + return createModel(url, null); + } + + public static ElasticInferenceServiceSparseEmbeddingsModel createModel(String url, Integer maxInputTokens) { + return new ElasticInferenceServiceSparseEmbeddingsModel( + "id", + TaskType.SPARSE_EMBEDDING, + "service", + new ElasticInferenceServiceSparseEmbeddingsServiceSettings(ElserModels.ELSER_V2_MODEL, maxInputTokens, null), + EmptyTaskSettings.INSTANCE, + EmptySecretSettings.INSTANCE, + new ElasticInferenceServiceComponents(url) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java new file mode 100644 index 0000000000000..a2b36cf9abdd5 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.elasticsearch.xpack.inference.services.elser.ElserModels; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.elser.ElserModelsTests.randomElserModel; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +public class ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase< + ElasticInferenceServiceSparseEmbeddingsServiceSettings> { + + @Override + protected Writeable.Reader instanceReader() { + return ElasticInferenceServiceSparseEmbeddingsServiceSettings::new; + } + + @Override + protected ElasticInferenceServiceSparseEmbeddingsServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected ElasticInferenceServiceSparseEmbeddingsServiceSettings mutateInstance( + ElasticInferenceServiceSparseEmbeddingsServiceSettings instance + ) throws IOException { + return randomValueOtherThan(instance, ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests::createRandom); + } + + public void testFromMap() { + var modelId = ElserModels.ELSER_V2_MODEL; + + var serviceSettings = ElasticInferenceServiceSparseEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId)), + ConfigurationParseContext.REQUEST + ); + + assertThat(serviceSettings, is(new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, null, null))); + } + + public void testFromMap_InvalidElserModelId() { + var invalidModelId = "invalid"; + + ValidationException validationException = expectThrows( + ValidationException.class, + () -> ElasticInferenceServiceSparseEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, invalidModelId)), + ConfigurationParseContext.REQUEST + ) + ); + + assertThat(validationException.getMessage(), containsString(Strings.format("unknown ELSER model id [%s]", invalidModelId))); + } + + public void testToXContent_WritesAlLFields() throws IOException { + var modelId = ElserModels.ELSER_V1_MODEL; + var maxInputTokens = 10; + var serviceSettings = new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, maxInputTokens, null); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + serviceSettings.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, is(Strings.format(""" + {"model_id":"%s","max_input_tokens":%d,"rate_limit":{"requests_per_minute":1000}}""", modelId, maxInputTokens))); + } + + public static ElasticInferenceServiceSparseEmbeddingsServiceSettings createRandom() { + return new ElasticInferenceServiceSparseEmbeddingsServiceSettings(randomElserModel(), randomNonNegativeInt(), null); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java new file mode 100644 index 0000000000000..62416f05800c6 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -0,0 +1,523 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elastic; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.EmptySecretSettings; +import org.elasticsearch.inference.EmptyTaskSettings; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests; +import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.elasticsearch.xpack.inference.services.elser.ElserModels; +import org.elasticsearch.xpack.inference.services.openai.OpenAiService; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; +import static org.elasticsearch.xpack.inference.Utils.getModelListenerForException; +import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; +import static org.elasticsearch.xpack.inference.Utils.getRequestConfigMap; +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class ElasticInferenceServiceTests extends ESTestCase { + + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testParseRequestConfig_CreatesASparseEmbeddingsModel() throws IOException { + try (var service = createServiceWithMockSender()) { + ActionListener modelListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class)); + + var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model; + assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL)); + + }, e -> fail("Model parsing should have succeeded, but failed: " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.SPARSE_EMBEDDING, + getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of()), + Set.of(), + modelListener + ); + } + } + + public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOException { + try (var service = createServiceWithMockSender()) { + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "The [elastic] service does not support task type [completion]" + ); + + service.parseRequestConfig( + "id", + TaskType.COMPLETION, + getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of()), + Set.of(), + failureListener + ); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createServiceWithMockSender()) { + var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of()); + config.put("extra_key", "value"); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service" + ); + service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInServiceSettingsMap() throws IOException { + try (var service = createServiceWithMockSender()) { + Map serviceSettings = new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)); + serviceSettings.put("extra_key", "value"); + + var config = getRequestConfigMap(serviceSettings, Map.of(), Map.of()); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service" + ); + service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInTaskSettingsMap() throws IOException { + try (var service = createServiceWithMockSender()) { + var taskSettings = Map.of("extra_key", (Object) "value"); + + var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), taskSettings, Map.of()); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service" + ); + service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInSecretSettingsMap() throws IOException { + try (var service = createServiceWithMockSender()) { + var secretSettings = Map.of("extra_key", (Object) "value"); + + var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), secretSettings); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service" + ); + service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesASparseEmbeddingModel() throws IOException { + try (var service = createServiceWithMockSender()) { + var persistedConfig = getPersistedConfigMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)), + Map.of(), + Map.of() + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class)); + + var sparseEmbeddingsModel = (ElasticInferenceServiceSparseEmbeddingsModel) model; + assertThat(sparseEmbeddingsModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL)); + assertThat(sparseEmbeddingsModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE)); + assertThat(sparseEmbeddingsModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE)); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createServiceWithMockSender()) { + var persistedConfig = getPersistedConfigMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)), + Map.of(), + Map.of() + ); + persistedConfig.config().put("extra_key", "value"); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class)); + + var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model; + assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL)); + assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE)); + assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE)); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException { + try (var service = createServiceWithMockSender()) { + Map serviceSettingsMap = new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)); + serviceSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap(serviceSettingsMap, Map.of(), Map.of()); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class)); + + var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model; + assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL)); + assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE)); + assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE)); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException { + try (var service = createServiceWithMockSender()) { + var taskSettings = Map.of("extra_key", (Object) "value"); + + var persistedConfig = getPersistedConfigMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)), + taskSettings, + Map.of() + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class)); + + var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model; + assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL)); + assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE)); + assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE)); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInSecretsSettings() throws IOException { + try (var service = createServiceWithMockSender()) { + var secretSettingsMap = Map.of("extra_key", (Object) "value"); + + var persistedConfig = getPersistedConfigMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)), + Map.of(), + secretSettingsMap + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class)); + + var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model; + assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL)); + assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE)); + assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE)); + } + } + + public void testCheckModelConfig_ReturnsNewModelReference() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer)); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + + var returnedModel = listener.actionGet(TIMEOUT); + assertThat(returnedModel, is(ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer)))); + } + } + + public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException { + var sender = mock(Sender.class); + + var factory = mock(HttpRequestSender.Factory.class); + when(factory.createSender()).thenReturn(sender); + + var mockModel = getInvalidModel("model_id", "service_name"); + + try ( + var service = new ElasticInferenceService( + factory, + createWithEmptySettings(threadPool), + new ElasticInferenceServiceComponents(null) + ) + ) { + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + mockModel, + null, + List.of(""), + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat( + thrownException.getMessage(), + is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.") + ); + + verify(factory, times(1)).createSender(); + verify(sender, times(1)).start(); + } + + verify(sender, times(1)).close(); + verifyNoMoreInteractions(factory); + verifyNoMoreInteractions(sender); + } + + public void testInfer_ThrowsWhenQueryIsPresent() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try ( + var service = new ElasticInferenceService( + senderFactory, + createWithEmptySettings(threadPool), + new ElasticInferenceServiceComponents(getUrl(webServer)) + ) + ) { + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer)); + + PlainActionFuture listener = new PlainActionFuture<>(); + UnsupportedOperationException exception = expectThrows( + UnsupportedOperationException.class, + () -> service.infer( + model, + "should throw", + List.of("abc"), + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ) + ); + + assertThat(exception.getMessage(), is("Query input not supported for Elastic Inference Service")); + } + } + + public void testInfer_SendsEmbeddingsRequest() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var eisGatewayUrl = getUrl(webServer); + + try ( + var service = new ElasticInferenceService( + senderFactory, + createWithEmptySettings(threadPool), + new ElasticInferenceServiceComponents(eisGatewayUrl) + ) + ) { + String responseJson = """ + { + "data": [ + { + "hello": 2.1259406, + "greet": 1.7073475 + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(eisGatewayUrl); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("input text"), + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + var result = listener.actionGet(TIMEOUT); + + assertThat( + result.asMap(), + Matchers.is( + SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings( + List.of( + new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), false) + ) + ) + ) + ); + var request = webServer.requests().get(0); + assertNull(request.getUri().getQuery()); + assertThat(request.getHeader(HttpHeaders.CONTENT_TYPE), Matchers.equalTo(XContentType.JSON.mediaType())); + + var requestMap = entityAsMap(request.getBody()); + assertThat(requestMap, is(Map.of("input", List.of("input text")))); + } + } + + public void testChunkedInfer_PassesThrough() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + var eisGatewayUrl = getUrl(webServer); + + try ( + var service = new ElasticInferenceService( + senderFactory, + createWithEmptySettings(threadPool), + new ElasticInferenceServiceComponents(eisGatewayUrl) + ) + ) { + String responseJson = """ + { + "data": [ + { + "hello": 2.1259406, + "greet": 1.7073475 + } + ] + } + """; + + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(eisGatewayUrl); + PlainActionFuture> listener = new PlainActionFuture<>(); + service.chunkedInfer( + model, + List.of("input text"), + new HashMap<>(), + InputType.INGEST, + new ChunkingOptions(null, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var results = listener.actionGet(TIMEOUT); + MatcherAssert.assertThat( + results.get(0).asMap(), + Matchers.is( + Map.of( + InferenceChunkedSparseEmbeddingResults.FIELD_NAME, + List.of( + Map.of( + ChunkedNlpInferenceResults.TEXT, + "input text", + ChunkedNlpInferenceResults.INFERENCE, + Map.of("hello", 2.1259406f, "greet", 1.7073475f) + ) + ) + ) + ) + ); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + assertThat(requestMap, is(Map.of("input", List.of("input text")))); + } + } + + private ElasticInferenceService createServiceWithMockSender() { + return new ElasticInferenceService( + mock(HttpRequestSender.Factory.class), + createWithEmptySettings(threadPool), + new ElasticInferenceServiceComponents(null) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java index ec753b9bec887..ffbdf1a5a6178 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java @@ -16,12 +16,12 @@ import java.io.IOException; import java.util.HashSet; +import static org.elasticsearch.xpack.inference.services.elser.ElserModelsTests.randomElserModel; + public class ElserInternalServiceSettingsTests extends AbstractWireSerializingTestCase { public static ElserInternalServiceSettings createRandom() { - return new ElserInternalServiceSettings( - ElasticsearchInternalServiceSettingsTests.validInstance(randomFrom(ElserInternalService.VALID_ELSER_MODEL_IDS)) - ); + return new ElserInternalServiceSettings(ElasticsearchInternalServiceSettingsTests.validInstance(randomElserModel())); } public void testBwcWrite() throws IOException { @@ -67,7 +67,7 @@ protected ElserInternalServiceSettings mutateInstance(ElserInternalServiceSettin ) ); case 2 -> { - var versions = new HashSet<>(ElserInternalService.VALID_ELSER_MODEL_IDS); + var versions = new HashSet<>(ElserModels.VALID_ELSER_MODEL_IDS); versions.remove(instance.modelId()); yield new ElserInternalServiceSettings( new ElasticsearchInternalServiceSettings( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java index f950e515a5336..85add1a0090c8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java @@ -171,7 +171,7 @@ public void testParseConfigStrictWithNoTaskSettings() { "foo", TaskType.SPARSE_EMBEDDING, ElserInternalService.NAME, - new ElserInternalServiceSettings(1, 4, ElserInternalService.ELSER_V2_MODEL, null), + new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null), ElserMlNodeTaskSettings.DEFAULT ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java new file mode 100644 index 0000000000000..f56e941dcc8c0 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elser; + +import org.elasticsearch.test.ESTestCase; + +public class ElserModelsTests extends ESTestCase { + + public static String randomElserModel() { + return randomFrom(ElserModels.VALID_ELSER_MODEL_IDS); + } + + public void testIsValidModel() { + assertTrue(ElserModels.isValidModel(randomElserModel())); + } + + public void testIsValidEisModel() { + assertTrue(ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL)); + } + + public void testIsInvalidModel() { + assertFalse(ElserModels.isValidModel("invalid")); + } + + public void testIsInvalidEisModel() { + assertFalse(ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL_LINUX_X86)); + } +} From 9f397a1ee13140bbb9774b5af7aecc8625c3765d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 9 Aug 2024 15:32:55 +0000 Subject: [PATCH 25/37] Bump versions after 8.15.0 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 12 ++++++------ .buildkite/pipelines/periodic.yml | 16 ++++++++-------- .ci/bwcVersions | 4 ++-- .ci/snapshotBwcVersions | 3 +-- .../org/elasticsearch/TransportVersions.java | 2 +- .../src/main/java/org/elasticsearch/Version.java | 2 +- .../org/elasticsearch/TransportVersions.csv | 1 + .../org/elasticsearch/index/IndexVersions.csv | 1 + 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index e323a9238ca5b..bb3c75f10aaea 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -62,7 +62,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.24", "8.14.4", "8.15.0", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 6e86e46f79484..12729a9b6ebda 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -577,8 +577,8 @@ steps: env: BWC_VERSION: 8.13.4 - - label: "{{matrix.image}} / 8.14.4 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.4 + - label: "{{matrix.image}} / 8.14.3 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.3 timeout_in_minutes: 300 matrix: setup: @@ -592,10 +592,10 @@ steps: buildDirectory: /dev/shm/bk diskSizeGb: 250 env: - BWC_VERSION: 8.14.4 + BWC_VERSION: 8.14.3 - - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0 + - label: "{{matrix.image}} / 8.15.1 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.1 timeout_in_minutes: 300 matrix: setup: @@ -609,7 +609,7 @@ steps: buildDirectory: /dev/shm/bk diskSizeGb: 250 env: - BWC_VERSION: 8.15.0 + BWC_VERSION: 8.15.1 - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index a0bc07f7ca3b7..740fec13d1790 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -642,8 +642,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.14.4 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.4#bwcTest + - label: 8.14.3 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.3#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -653,7 +653,7 @@ steps: preemptible: true diskSizeGb: 250 env: - BWC_VERSION: 8.14.4 + BWC_VERSION: 8.14.3 retry: automatic: - exit_status: "-1" @@ -662,8 +662,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.15.0 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.0#bwcTest + - label: 8.15.1 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.1#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -673,7 +673,7 @@ steps: preemptible: true diskSizeGb: 250 env: - BWC_VERSION: 8.15.0 + BWC_VERSION: 8.15.1 retry: automatic: - exit_status: "-1" @@ -771,7 +771,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.24", "8.14.4", "8.15.0", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -821,7 +821,7 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.24", "8.14.4", "8.15.0", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index d6072488ae93b..e43b3333dd755 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -31,6 +31,6 @@ BWC_VERSION: - "8.11.4" - "8.12.2" - "8.13.4" - - "8.14.4" - - "8.15.0" + - "8.14.3" + - "8.15.1" - "8.16.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 909960a67cc41..2eea118e57e2a 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,4 @@ BWC_VERSION: - "7.17.24" - - "8.14.4" - - "8.15.0" + - "8.15.1" - "8.16.0" diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 2b579894ca521..930adaf6258d1 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -251,7 +251,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = V_8_14_0; + public static final TransportVersion MINIMUM_CCS_VERSION = FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index fd29a81cdb143..333669ca8079c 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -180,8 +180,8 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_14_1 = new Version(8_14_01_99); public static final Version V_8_14_2 = new Version(8_14_02_99); public static final Version V_8_14_3 = new Version(8_14_03_99); - public static final Version V_8_14_4 = new Version(8_14_04_99); public static final Version V_8_15_0 = new Version(8_15_00_99); + public static final Version V_8_15_1 = new Version(8_15_01_99); public static final Version V_8_16_0 = new Version(8_16_00_99); public static final Version CURRENT = V_8_16_0; diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index 687e435990785..8cd6fe9720039 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -126,3 +126,4 @@ 8.14.1,8636001 8.14.2,8636001 8.14.3,8636001 +8.15.0,8702002 diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index 8c86ca48d6284..df0df2d05ba5b 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -126,3 +126,4 @@ 8.14.1,8505000 8.14.2,8505000 8.14.3,8505000 +8.15.0,8512000 From bceac553133b6fe7855afe57840022c05c904195 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 9 Aug 2024 15:33:49 +0000 Subject: [PATCH 26/37] Prune changelogs after 8.15.0 release --- docs/changelog/101373.yaml | 6 ------ docs/changelog/103374.yaml | 16 ---------------- docs/changelog/105792.yaml | 18 ------------------ docs/changelog/105829.yaml | 5 ----- docs/changelog/106252.yaml | 6 ------ docs/changelog/106486.yaml | 17 ----------------- docs/changelog/106553.yaml | 5 ----- docs/changelog/106591.yaml | 5 ----- docs/changelog/106820.yaml | 5 ----- docs/changelog/107081.yaml | 5 ----- docs/changelog/107088.yaml | 5 ----- docs/changelog/107191.yaml | 17 ----------------- docs/changelog/107216.yaml | 5 ----- docs/changelog/107240.yaml | 6 ------ docs/changelog/107244.yaml | 5 ----- docs/changelog/107279.yaml | 5 ----- docs/changelog/107409.yaml | 5 ----- docs/changelog/107410.yaml | 5 ----- docs/changelog/107415.yaml | 6 ------ docs/changelog/107426.yaml | 5 ----- docs/changelog/107435.yaml | 6 ------ docs/changelog/107493.yaml | 5 ----- docs/changelog/107545.yaml | 6 ------ docs/changelog/107549.yaml | 5 ----- docs/changelog/107567.yaml | 5 ----- docs/changelog/107579.yaml | 6 ------ docs/changelog/107593.yaml | 5 ----- docs/changelog/107640.yaml | 6 ------ docs/changelog/107645.yaml | 7 ------- docs/changelog/107647.yaml | 5 ----- docs/changelog/107663.yaml | 5 ----- docs/changelog/107675.yaml | 17 ----------------- docs/changelog/107676.yaml | 5 ----- docs/changelog/107706.yaml | 5 ----- docs/changelog/107735.yaml | 5 ----- docs/changelog/107739.yaml | 6 ------ docs/changelog/107764.yaml | 5 ----- docs/changelog/107779.yaml | 6 ------ docs/changelog/107792.yaml | 5 ----- docs/changelog/107813.yaml | 6 ------ docs/changelog/107827.yaml | 5 ----- docs/changelog/107832.yaml | 5 ----- docs/changelog/107862.yaml | 6 ------ docs/changelog/107876.yaml | 5 ----- docs/changelog/107877.yaml | 5 ----- docs/changelog/107886.yaml | 5 ----- docs/changelog/107892.yaml | 5 ----- docs/changelog/107893.yaml | 5 ----- docs/changelog/107897.yaml | 5 ----- docs/changelog/107917.yaml | 6 ------ docs/changelog/107922.yaml | 6 ------ docs/changelog/107930.yaml | 5 ----- docs/changelog/107937.yaml | 5 ----- docs/changelog/107947.yaml | 6 ------ docs/changelog/107967.yaml | 6 ------ docs/changelog/107972.yaml | 5 ----- docs/changelog/107977.yaml | 6 ------ docs/changelog/107978.yaml | 6 ------ docs/changelog/107987.yaml | 6 ------ docs/changelog/107990.yaml | 5 ----- docs/changelog/108016.yaml | 5 ----- docs/changelog/108019.yaml | 6 ------ docs/changelog/108051.yaml | 5 ----- docs/changelog/108065.yaml | 5 ----- docs/changelog/108070.yaml | 5 ----- docs/changelog/108088.yaml | 5 ----- docs/changelog/108089.yaml | 6 ------ docs/changelog/108106.yaml | 6 ------ docs/changelog/108118.yaml | 5 ----- docs/changelog/108122.yaml | 6 ------ docs/changelog/108130.yaml | 5 ----- docs/changelog/108131.yaml | 5 ----- docs/changelog/108144.yaml | 5 ----- docs/changelog/108145.yaml | 5 ----- docs/changelog/108146.yaml | 5 ----- docs/changelog/108155.yaml | 5 ----- docs/changelog/108161.yaml | 5 ----- docs/changelog/108165.yaml | 5 ----- docs/changelog/108171.yaml | 5 ----- docs/changelog/108222.yaml | 5 ----- docs/changelog/108223.yaml | 5 ----- docs/changelog/108227.yaml | 5 ----- docs/changelog/108254.yaml | 5 ----- docs/changelog/108266.yaml | 5 ----- docs/changelog/108300.yaml | 5 ----- docs/changelog/108306.yaml | 5 ----- docs/changelog/108333.yaml | 5 ----- docs/changelog/108340.yaml | 5 ----- docs/changelog/108349.yaml | 6 ------ docs/changelog/108360.yaml | 6 ------ docs/changelog/108379.yaml | 5 ----- docs/changelog/108394.yaml | 6 ------ docs/changelog/108395.yaml | 5 ----- docs/changelog/108396.yaml | 6 ------ docs/changelog/108409.yaml | 6 ------ docs/changelog/108410.yaml | 5 ----- docs/changelog/108417.yaml | 6 ------ docs/changelog/108421.yaml | 6 ------ docs/changelog/108429.yaml | 6 ------ docs/changelog/108444.yaml | 5 ----- docs/changelog/108452.yaml | 5 ----- docs/changelog/108455.yaml | 6 ------ docs/changelog/108459.yaml | 6 ------ docs/changelog/108472.yaml | 5 ----- docs/changelog/108517.yaml | 6 ------ docs/changelog/108521.yaml | 6 ------ docs/changelog/108522.yaml | 5 ----- docs/changelog/108537.yaml | 6 ------ docs/changelog/108538.yaml | 5 ----- docs/changelog/108574.yaml | 5 ----- docs/changelog/108602.yaml | 5 ----- docs/changelog/108606.yaml | 14 -------------- docs/changelog/108607.yaml | 5 ----- docs/changelog/108612.yaml | 5 ----- docs/changelog/108624.yaml | 12 ------------ docs/changelog/108639.yaml | 5 ----- docs/changelog/108643.yaml | 6 ------ docs/changelog/108651.yaml | 5 ----- docs/changelog/108672.yaml | 5 ----- docs/changelog/108679.yaml | 6 ------ docs/changelog/108682.yaml | 5 ----- docs/changelog/108683.yaml | 14 -------------- docs/changelog/108684.yaml | 5 ----- docs/changelog/108687.yaml | 5 ----- docs/changelog/108693.yaml | 5 ----- docs/changelog/108705.yaml | 6 ------ docs/changelog/108713.yaml | 6 ------ docs/changelog/108726.yaml | 5 ----- docs/changelog/108733.yaml | 5 ----- docs/changelog/108746.yaml | 5 ----- docs/changelog/108759.yaml | 5 ----- docs/changelog/108761.yaml | 5 ----- docs/changelog/108764.yaml | 6 ------ docs/changelog/108780.yaml | 6 ------ docs/changelog/108786.yaml | 5 ----- docs/changelog/108793.yaml | 5 ----- docs/changelog/108796.yaml | 5 ----- docs/changelog/108814.yaml | 6 ------ docs/changelog/108818.yaml | 5 ----- docs/changelog/108820.yaml | 5 ----- docs/changelog/108822.yaml | 6 ------ docs/changelog/108831.yaml | 5 ----- docs/changelog/108849.yaml | 6 ------ docs/changelog/108856.yaml | 5 ----- docs/changelog/108860.yaml | 5 ----- docs/changelog/108862.yaml | 5 ----- docs/changelog/108868.yaml | 5 ----- docs/changelog/108870.yaml | 5 ----- docs/changelog/108871.yaml | 5 ----- docs/changelog/108878.yaml | 5 ----- docs/changelog/108881.yaml | 5 ----- docs/changelog/108885.yaml | 5 ----- docs/changelog/108886.yaml | 5 ----- docs/changelog/108891.yaml | 6 ------ docs/changelog/108895.yaml | 5 ----- docs/changelog/108896.yaml | 6 ------ docs/changelog/108911.yaml | 5 ----- docs/changelog/108942.yaml | 5 ----- docs/changelog/108947.yaml | 5 ----- docs/changelog/108999.yaml | 5 ----- docs/changelog/109007.yaml | 5 ----- docs/changelog/109025.yaml | 6 ------ docs/changelog/109042.yaml | 5 ----- docs/changelog/109043.yaml | 5 ----- docs/changelog/109044.yaml | 5 ----- docs/changelog/109047.yaml | 5 ----- docs/changelog/109070.yaml | 6 ------ docs/changelog/109071.yaml | 5 ----- docs/changelog/109078.yaml | 5 ----- docs/changelog/109084.yaml | 5 ----- docs/changelog/109104.yaml | 6 ------ docs/changelog/109123.yaml | 5 ----- docs/changelog/109126.yaml | 5 ----- docs/changelog/109167.yaml | 5 ----- docs/changelog/109174.yaml | 5 ----- docs/changelog/109185.yaml | 6 ------ docs/changelog/109194.yaml | 5 ----- docs/changelog/109196.yaml | 5 ----- docs/changelog/109204.yaml | 5 ----- docs/changelog/109205.yaml | 6 ------ docs/changelog/109219.yaml | 15 --------------- docs/changelog/109220.yaml | 5 ----- docs/changelog/109233.yaml | 5 ----- docs/changelog/109236.yaml | 6 ------ docs/changelog/109240.yaml | 5 ----- docs/changelog/109241.yaml | 5 ----- docs/changelog/109256.yaml | 7 ------- docs/changelog/109312.yaml | 5 ----- docs/changelog/109317.yaml | 13 ------------- docs/changelog/109332.yaml | 5 ----- docs/changelog/109358.yaml | 5 ----- docs/changelog/109359.yaml | 5 ----- docs/changelog/109370.yaml | 6 ------ docs/changelog/109384.yaml | 5 ----- docs/changelog/109386.yaml | 6 ------ docs/changelog/109395.yaml | 5 ----- docs/changelog/109410.yaml | 5 ----- docs/changelog/109444.yaml | 5 ----- docs/changelog/109449.yaml | 6 ------ docs/changelog/109462.yaml | 6 ------ docs/changelog/109470.yaml | 5 ----- docs/changelog/109480.yaml | 5 ----- docs/changelog/109481.yaml | 5 ----- docs/changelog/109487.yaml | 5 ----- docs/changelog/109501.yaml | 14 -------------- docs/changelog/109506.yaml | 6 ------ docs/changelog/109534.yaml | 6 ------ docs/changelog/109540.yaml | 6 ------ docs/changelog/109551.yaml | 5 ----- docs/changelog/109554.yaml | 6 ------ docs/changelog/109563.yaml | 5 ----- docs/changelog/109597.yaml | 5 ----- docs/changelog/109603.yaml | 5 ----- docs/changelog/109606.yaml | 5 ----- docs/changelog/109613.yaml | 6 ------ docs/changelog/109618.yaml | 6 ------ docs/changelog/109634.yaml | 5 ----- docs/changelog/109651.yaml | 5 ----- docs/changelog/109653.yaml | 5 ----- docs/changelog/109657.yaml | 5 ----- docs/changelog/109672.yaml | 5 ----- docs/changelog/109717.yaml | 5 ----- docs/changelog/109720.yaml | 5 ----- docs/changelog/109746.yaml | 6 ------ docs/changelog/109779.yaml | 5 ----- docs/changelog/109781.yaml | 5 ----- docs/changelog/109794.yaml | 5 ----- docs/changelog/109807.yaml | 6 ------ docs/changelog/109813.yaml | 5 ----- docs/changelog/109848.yaml | 5 ----- docs/changelog/109873.yaml | 5 ----- docs/changelog/109876.yaml | 6 ------ docs/changelog/109880.yaml | 10 ---------- docs/changelog/109882.yaml | 5 ----- docs/changelog/109893.yaml | 5 ----- docs/changelog/109908.yaml | 5 ----- docs/changelog/109931.yaml | 5 ----- docs/changelog/109957.yaml | 6 ------ docs/changelog/109963.yaml | 6 ------ docs/changelog/109967.yaml | 5 ----- docs/changelog/109981.yaml | 5 ----- docs/changelog/109989.yaml | 5 ----- docs/changelog/109993.yaml | 5 ----- docs/changelog/110004.yaml | 11 ----------- docs/changelog/110016.yaml | 5 ----- docs/changelog/110019.yaml | 6 ------ docs/changelog/110046.yaml | 6 ------ docs/changelog/110059.yaml | 32 -------------------------------- docs/changelog/110061.yaml | 6 ------ docs/changelog/110066.yaml | 6 ------ docs/changelog/110096.yaml | 6 ------ docs/changelog/110102.yaml | 6 ------ docs/changelog/110112.yaml | 5 ----- docs/changelog/110146.yaml | 5 ----- docs/changelog/110160.yaml | 5 ----- docs/changelog/110176.yaml | 5 ----- docs/changelog/110177.yaml | 5 ----- docs/changelog/110179.yaml | 6 ------ docs/changelog/110186.yaml | 6 ------ docs/changelog/110201.yaml | 6 ------ docs/changelog/110214.yaml | 5 ----- docs/changelog/110233.yaml | 6 ------ docs/changelog/110234.yaml | 5 ----- docs/changelog/110236.yaml | 21 --------------------- docs/changelog/110248.yaml | 5 ----- docs/changelog/110251.yaml | 13 ------------- docs/changelog/110334.yaml | 5 ----- docs/changelog/110337.yaml | 5 ----- docs/changelog/110338.yaml | 5 ----- docs/changelog/110347.yaml | 5 ----- docs/changelog/110361.yaml | 7 ------- docs/changelog/110369.yaml | 6 ------ docs/changelog/110383.yaml | 5 ----- docs/changelog/110391.yaml | 6 ------ docs/changelog/110395.yaml | 9 --------- docs/changelog/110431.yaml | 5 ----- docs/changelog/110476.yaml | 7 ------- docs/changelog/110488.yaml | 6 ------ docs/changelog/110540.yaml | 16 ---------------- docs/changelog/110586.yaml | 5 ----- docs/changelog/110651.yaml | 5 ----- docs/changelog/110665.yaml | 6 ------ docs/changelog/110666.yaml | 5 ----- docs/changelog/110707.yaml | 5 ----- docs/changelog/110710.yaml | 6 ------ docs/changelog/110793.yaml | 7 ------- docs/changelog/110824.yaml | 5 ----- docs/changelog/110844.yaml | 5 ----- docs/changelog/110906.yaml | 5 ----- docs/changelog/110922.yaml | 5 ----- docs/changelog/110927.yaml | 5 ----- docs/changelog/111184.yaml | 5 ----- docs/changelog/111186.yaml | 6 ------ docs/changelog/111290.yaml | 5 ----- docs/changelog/111363.yaml | 6 ------ docs/changelog/111366.yaml | 6 ------ docs/changelog/111369.yaml | 5 ----- 297 files changed, 1770 deletions(-) delete mode 100644 docs/changelog/101373.yaml delete mode 100644 docs/changelog/103374.yaml delete mode 100644 docs/changelog/105792.yaml delete mode 100644 docs/changelog/105829.yaml delete mode 100644 docs/changelog/106252.yaml delete mode 100644 docs/changelog/106486.yaml delete mode 100644 docs/changelog/106553.yaml delete mode 100644 docs/changelog/106591.yaml delete mode 100644 docs/changelog/106820.yaml delete mode 100644 docs/changelog/107081.yaml delete mode 100644 docs/changelog/107088.yaml delete mode 100644 docs/changelog/107191.yaml delete mode 100644 docs/changelog/107216.yaml delete mode 100644 docs/changelog/107240.yaml delete mode 100644 docs/changelog/107244.yaml delete mode 100644 docs/changelog/107279.yaml delete mode 100644 docs/changelog/107409.yaml delete mode 100644 docs/changelog/107410.yaml delete mode 100644 docs/changelog/107415.yaml delete mode 100644 docs/changelog/107426.yaml delete mode 100644 docs/changelog/107435.yaml delete mode 100644 docs/changelog/107493.yaml delete mode 100644 docs/changelog/107545.yaml delete mode 100644 docs/changelog/107549.yaml delete mode 100644 docs/changelog/107567.yaml delete mode 100644 docs/changelog/107579.yaml delete mode 100644 docs/changelog/107593.yaml delete mode 100644 docs/changelog/107640.yaml delete mode 100644 docs/changelog/107645.yaml delete mode 100644 docs/changelog/107647.yaml delete mode 100644 docs/changelog/107663.yaml delete mode 100644 docs/changelog/107675.yaml delete mode 100644 docs/changelog/107676.yaml delete mode 100644 docs/changelog/107706.yaml delete mode 100644 docs/changelog/107735.yaml delete mode 100644 docs/changelog/107739.yaml delete mode 100644 docs/changelog/107764.yaml delete mode 100644 docs/changelog/107779.yaml delete mode 100644 docs/changelog/107792.yaml delete mode 100644 docs/changelog/107813.yaml delete mode 100644 docs/changelog/107827.yaml delete mode 100644 docs/changelog/107832.yaml delete mode 100644 docs/changelog/107862.yaml delete mode 100644 docs/changelog/107876.yaml delete mode 100644 docs/changelog/107877.yaml delete mode 100644 docs/changelog/107886.yaml delete mode 100644 docs/changelog/107892.yaml delete mode 100644 docs/changelog/107893.yaml delete mode 100644 docs/changelog/107897.yaml delete mode 100644 docs/changelog/107917.yaml delete mode 100644 docs/changelog/107922.yaml delete mode 100644 docs/changelog/107930.yaml delete mode 100644 docs/changelog/107937.yaml delete mode 100644 docs/changelog/107947.yaml delete mode 100644 docs/changelog/107967.yaml delete mode 100644 docs/changelog/107972.yaml delete mode 100644 docs/changelog/107977.yaml delete mode 100644 docs/changelog/107978.yaml delete mode 100644 docs/changelog/107987.yaml delete mode 100644 docs/changelog/107990.yaml delete mode 100644 docs/changelog/108016.yaml delete mode 100644 docs/changelog/108019.yaml delete mode 100644 docs/changelog/108051.yaml delete mode 100644 docs/changelog/108065.yaml delete mode 100644 docs/changelog/108070.yaml delete mode 100644 docs/changelog/108088.yaml delete mode 100644 docs/changelog/108089.yaml delete mode 100644 docs/changelog/108106.yaml delete mode 100644 docs/changelog/108118.yaml delete mode 100644 docs/changelog/108122.yaml delete mode 100644 docs/changelog/108130.yaml delete mode 100644 docs/changelog/108131.yaml delete mode 100644 docs/changelog/108144.yaml delete mode 100644 docs/changelog/108145.yaml delete mode 100644 docs/changelog/108146.yaml delete mode 100644 docs/changelog/108155.yaml delete mode 100644 docs/changelog/108161.yaml delete mode 100644 docs/changelog/108165.yaml delete mode 100644 docs/changelog/108171.yaml delete mode 100644 docs/changelog/108222.yaml delete mode 100644 docs/changelog/108223.yaml delete mode 100644 docs/changelog/108227.yaml delete mode 100644 docs/changelog/108254.yaml delete mode 100644 docs/changelog/108266.yaml delete mode 100644 docs/changelog/108300.yaml delete mode 100644 docs/changelog/108306.yaml delete mode 100644 docs/changelog/108333.yaml delete mode 100644 docs/changelog/108340.yaml delete mode 100644 docs/changelog/108349.yaml delete mode 100644 docs/changelog/108360.yaml delete mode 100644 docs/changelog/108379.yaml delete mode 100644 docs/changelog/108394.yaml delete mode 100644 docs/changelog/108395.yaml delete mode 100644 docs/changelog/108396.yaml delete mode 100644 docs/changelog/108409.yaml delete mode 100644 docs/changelog/108410.yaml delete mode 100644 docs/changelog/108417.yaml delete mode 100644 docs/changelog/108421.yaml delete mode 100644 docs/changelog/108429.yaml delete mode 100644 docs/changelog/108444.yaml delete mode 100644 docs/changelog/108452.yaml delete mode 100644 docs/changelog/108455.yaml delete mode 100644 docs/changelog/108459.yaml delete mode 100644 docs/changelog/108472.yaml delete mode 100644 docs/changelog/108517.yaml delete mode 100644 docs/changelog/108521.yaml delete mode 100644 docs/changelog/108522.yaml delete mode 100644 docs/changelog/108537.yaml delete mode 100644 docs/changelog/108538.yaml delete mode 100644 docs/changelog/108574.yaml delete mode 100644 docs/changelog/108602.yaml delete mode 100644 docs/changelog/108606.yaml delete mode 100644 docs/changelog/108607.yaml delete mode 100644 docs/changelog/108612.yaml delete mode 100644 docs/changelog/108624.yaml delete mode 100644 docs/changelog/108639.yaml delete mode 100644 docs/changelog/108643.yaml delete mode 100644 docs/changelog/108651.yaml delete mode 100644 docs/changelog/108672.yaml delete mode 100644 docs/changelog/108679.yaml delete mode 100644 docs/changelog/108682.yaml delete mode 100644 docs/changelog/108683.yaml delete mode 100644 docs/changelog/108684.yaml delete mode 100644 docs/changelog/108687.yaml delete mode 100644 docs/changelog/108693.yaml delete mode 100644 docs/changelog/108705.yaml delete mode 100644 docs/changelog/108713.yaml delete mode 100644 docs/changelog/108726.yaml delete mode 100644 docs/changelog/108733.yaml delete mode 100644 docs/changelog/108746.yaml delete mode 100644 docs/changelog/108759.yaml delete mode 100644 docs/changelog/108761.yaml delete mode 100644 docs/changelog/108764.yaml delete mode 100644 docs/changelog/108780.yaml delete mode 100644 docs/changelog/108786.yaml delete mode 100644 docs/changelog/108793.yaml delete mode 100644 docs/changelog/108796.yaml delete mode 100644 docs/changelog/108814.yaml delete mode 100644 docs/changelog/108818.yaml delete mode 100644 docs/changelog/108820.yaml delete mode 100644 docs/changelog/108822.yaml delete mode 100644 docs/changelog/108831.yaml delete mode 100644 docs/changelog/108849.yaml delete mode 100644 docs/changelog/108856.yaml delete mode 100644 docs/changelog/108860.yaml delete mode 100644 docs/changelog/108862.yaml delete mode 100644 docs/changelog/108868.yaml delete mode 100644 docs/changelog/108870.yaml delete mode 100644 docs/changelog/108871.yaml delete mode 100644 docs/changelog/108878.yaml delete mode 100644 docs/changelog/108881.yaml delete mode 100644 docs/changelog/108885.yaml delete mode 100644 docs/changelog/108886.yaml delete mode 100644 docs/changelog/108891.yaml delete mode 100644 docs/changelog/108895.yaml delete mode 100644 docs/changelog/108896.yaml delete mode 100644 docs/changelog/108911.yaml delete mode 100644 docs/changelog/108942.yaml delete mode 100644 docs/changelog/108947.yaml delete mode 100644 docs/changelog/108999.yaml delete mode 100644 docs/changelog/109007.yaml delete mode 100644 docs/changelog/109025.yaml delete mode 100644 docs/changelog/109042.yaml delete mode 100644 docs/changelog/109043.yaml delete mode 100644 docs/changelog/109044.yaml delete mode 100644 docs/changelog/109047.yaml delete mode 100644 docs/changelog/109070.yaml delete mode 100644 docs/changelog/109071.yaml delete mode 100644 docs/changelog/109078.yaml delete mode 100644 docs/changelog/109084.yaml delete mode 100644 docs/changelog/109104.yaml delete mode 100644 docs/changelog/109123.yaml delete mode 100644 docs/changelog/109126.yaml delete mode 100644 docs/changelog/109167.yaml delete mode 100644 docs/changelog/109174.yaml delete mode 100644 docs/changelog/109185.yaml delete mode 100644 docs/changelog/109194.yaml delete mode 100644 docs/changelog/109196.yaml delete mode 100644 docs/changelog/109204.yaml delete mode 100644 docs/changelog/109205.yaml delete mode 100644 docs/changelog/109219.yaml delete mode 100644 docs/changelog/109220.yaml delete mode 100644 docs/changelog/109233.yaml delete mode 100644 docs/changelog/109236.yaml delete mode 100644 docs/changelog/109240.yaml delete mode 100644 docs/changelog/109241.yaml delete mode 100644 docs/changelog/109256.yaml delete mode 100644 docs/changelog/109312.yaml delete mode 100644 docs/changelog/109317.yaml delete mode 100644 docs/changelog/109332.yaml delete mode 100644 docs/changelog/109358.yaml delete mode 100644 docs/changelog/109359.yaml delete mode 100644 docs/changelog/109370.yaml delete mode 100644 docs/changelog/109384.yaml delete mode 100644 docs/changelog/109386.yaml delete mode 100644 docs/changelog/109395.yaml delete mode 100644 docs/changelog/109410.yaml delete mode 100644 docs/changelog/109444.yaml delete mode 100644 docs/changelog/109449.yaml delete mode 100644 docs/changelog/109462.yaml delete mode 100644 docs/changelog/109470.yaml delete mode 100644 docs/changelog/109480.yaml delete mode 100644 docs/changelog/109481.yaml delete mode 100644 docs/changelog/109487.yaml delete mode 100644 docs/changelog/109501.yaml delete mode 100644 docs/changelog/109506.yaml delete mode 100644 docs/changelog/109534.yaml delete mode 100644 docs/changelog/109540.yaml delete mode 100644 docs/changelog/109551.yaml delete mode 100644 docs/changelog/109554.yaml delete mode 100644 docs/changelog/109563.yaml delete mode 100644 docs/changelog/109597.yaml delete mode 100644 docs/changelog/109603.yaml delete mode 100644 docs/changelog/109606.yaml delete mode 100644 docs/changelog/109613.yaml delete mode 100644 docs/changelog/109618.yaml delete mode 100644 docs/changelog/109634.yaml delete mode 100644 docs/changelog/109651.yaml delete mode 100644 docs/changelog/109653.yaml delete mode 100644 docs/changelog/109657.yaml delete mode 100644 docs/changelog/109672.yaml delete mode 100644 docs/changelog/109717.yaml delete mode 100644 docs/changelog/109720.yaml delete mode 100644 docs/changelog/109746.yaml delete mode 100644 docs/changelog/109779.yaml delete mode 100644 docs/changelog/109781.yaml delete mode 100644 docs/changelog/109794.yaml delete mode 100644 docs/changelog/109807.yaml delete mode 100644 docs/changelog/109813.yaml delete mode 100644 docs/changelog/109848.yaml delete mode 100644 docs/changelog/109873.yaml delete mode 100644 docs/changelog/109876.yaml delete mode 100644 docs/changelog/109880.yaml delete mode 100644 docs/changelog/109882.yaml delete mode 100644 docs/changelog/109893.yaml delete mode 100644 docs/changelog/109908.yaml delete mode 100644 docs/changelog/109931.yaml delete mode 100644 docs/changelog/109957.yaml delete mode 100644 docs/changelog/109963.yaml delete mode 100644 docs/changelog/109967.yaml delete mode 100644 docs/changelog/109981.yaml delete mode 100644 docs/changelog/109989.yaml delete mode 100644 docs/changelog/109993.yaml delete mode 100644 docs/changelog/110004.yaml delete mode 100644 docs/changelog/110016.yaml delete mode 100644 docs/changelog/110019.yaml delete mode 100644 docs/changelog/110046.yaml delete mode 100644 docs/changelog/110059.yaml delete mode 100644 docs/changelog/110061.yaml delete mode 100644 docs/changelog/110066.yaml delete mode 100644 docs/changelog/110096.yaml delete mode 100644 docs/changelog/110102.yaml delete mode 100644 docs/changelog/110112.yaml delete mode 100644 docs/changelog/110146.yaml delete mode 100644 docs/changelog/110160.yaml delete mode 100644 docs/changelog/110176.yaml delete mode 100644 docs/changelog/110177.yaml delete mode 100644 docs/changelog/110179.yaml delete mode 100644 docs/changelog/110186.yaml delete mode 100644 docs/changelog/110201.yaml delete mode 100644 docs/changelog/110214.yaml delete mode 100644 docs/changelog/110233.yaml delete mode 100644 docs/changelog/110234.yaml delete mode 100644 docs/changelog/110236.yaml delete mode 100644 docs/changelog/110248.yaml delete mode 100644 docs/changelog/110251.yaml delete mode 100644 docs/changelog/110334.yaml delete mode 100644 docs/changelog/110337.yaml delete mode 100644 docs/changelog/110338.yaml delete mode 100644 docs/changelog/110347.yaml delete mode 100644 docs/changelog/110361.yaml delete mode 100644 docs/changelog/110369.yaml delete mode 100644 docs/changelog/110383.yaml delete mode 100644 docs/changelog/110391.yaml delete mode 100644 docs/changelog/110395.yaml delete mode 100644 docs/changelog/110431.yaml delete mode 100644 docs/changelog/110476.yaml delete mode 100644 docs/changelog/110488.yaml delete mode 100644 docs/changelog/110540.yaml delete mode 100644 docs/changelog/110586.yaml delete mode 100644 docs/changelog/110651.yaml delete mode 100644 docs/changelog/110665.yaml delete mode 100644 docs/changelog/110666.yaml delete mode 100644 docs/changelog/110707.yaml delete mode 100644 docs/changelog/110710.yaml delete mode 100644 docs/changelog/110793.yaml delete mode 100644 docs/changelog/110824.yaml delete mode 100644 docs/changelog/110844.yaml delete mode 100644 docs/changelog/110906.yaml delete mode 100644 docs/changelog/110922.yaml delete mode 100644 docs/changelog/110927.yaml delete mode 100644 docs/changelog/111184.yaml delete mode 100644 docs/changelog/111186.yaml delete mode 100644 docs/changelog/111290.yaml delete mode 100644 docs/changelog/111363.yaml delete mode 100644 docs/changelog/111366.yaml delete mode 100644 docs/changelog/111369.yaml diff --git a/docs/changelog/101373.yaml b/docs/changelog/101373.yaml deleted file mode 100644 index 53b5680301c79..0000000000000 --- a/docs/changelog/101373.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 101373 -summary: Adding aggregations support for the `_ignored` field -area: Search -type: feature -issues: - - 59946 diff --git a/docs/changelog/103374.yaml b/docs/changelog/103374.yaml deleted file mode 100644 index fcdee9185eb92..0000000000000 --- a/docs/changelog/103374.yaml +++ /dev/null @@ -1,16 +0,0 @@ -pr: 103374 -summary: Cut over stored fields to ZSTD for compression -area: Search -type: enhancement -issues: [] -highlight: - title: Stored fields are now compressed with ZStandard instead of LZ4/DEFLATE - body: |- - Stored fields are now compressed by splitting documents into blocks, which - are then compressed independently with ZStandard. `index.codec: default` - (default) uses blocks of at most 14kB or 128 documents compressed with level - 0, while `index.codec: best_compression` uses blocks of at most 240kB or - 2048 documents compressed at level 3. On most datasets that we tested - against, this yielded storage improvements in the order of 10%, slightly - faster indexing and similar retrieval latencies. - notable: true diff --git a/docs/changelog/105792.yaml b/docs/changelog/105792.yaml deleted file mode 100644 index b9190e60cc96d..0000000000000 --- a/docs/changelog/105792.yaml +++ /dev/null @@ -1,18 +0,0 @@ -pr: 105792 -summary: "Change `skip_unavailable` remote cluster setting default value to true" -area: Search -type: breaking -issues: [] -breaking: - title: "Change `skip_unavailable` remote cluster setting default value to true" - area: Cluster and node setting - details: The default value of the `skip_unavailable` setting is now set to true. - All existing and future remote clusters that do not define this setting will use the new default. - This setting only affects cross-cluster searches using the _search or _async_search API. - impact: Unavailable remote clusters in a cross-cluster search will no longer cause the search to fail unless - skip_unavailable is configured to be `false` in elasticsearch.yml or via the `_cluster/settings` API. - Unavailable clusters with `skip_unavailable`=`true` (either explicitly or by using the new default) are marked - as SKIPPED in the search response metadata section and do not fail the entire search. If users want to ensure that a - search returns a failure when a particular remote cluster is not available, `skip_unavailable` must be now be - set explicitly. - notable: true diff --git a/docs/changelog/105829.yaml b/docs/changelog/105829.yaml deleted file mode 100644 index d9f8439e4b887..0000000000000 --- a/docs/changelog/105829.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 105829 -summary: Log shard movements -area: Allocation -type: enhancement -issues: [] diff --git a/docs/changelog/106252.yaml b/docs/changelog/106252.yaml deleted file mode 100644 index 5e3f084632b9d..0000000000000 --- a/docs/changelog/106252.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 106252 -summary: Add min/max range of the `event.ingested` field to cluster state for searchable - snapshots -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/106486.yaml b/docs/changelog/106486.yaml deleted file mode 100644 index b33df50780e02..0000000000000 --- a/docs/changelog/106486.yaml +++ /dev/null @@ -1,17 +0,0 @@ -pr: 106486 -summary: Create custom parser for ISO-8601 datetimes -area: Infra/Core -type: enhancement -issues: - - 102063 -highlight: - title: New custom parser for ISO-8601 datetimes - body: |- - This introduces a new custom parser for ISO-8601 datetimes, for the `iso8601`, `strict_date_optional_time`, and - `strict_date_optional_time_nanos` built-in date formats. This provides a performance improvement over the - default Java date-time parsing. Whilst it maintains much of the same behaviour, - the new parser does not accept nonsensical date-time strings that have multiple fractional seconds fields - or multiple timezone specifiers. If the new parser fails to parse a string, it will then use the previous parser - to parse it. If a large proportion of the input data consists of these invalid strings, this may cause - a small performance degradation. If you wish to force the use of the old parsers regardless, - set the JVM property `es.datetime.java_time_parsers=true` on all ES nodes. diff --git a/docs/changelog/106553.yaml b/docs/changelog/106553.yaml deleted file mode 100644 index 0ec5b1bb02da8..0000000000000 --- a/docs/changelog/106553.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 106553 -summary: Add support for hiragana_uppercase & katakana_uppercase token filters in kuromoji analysis plugin -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/106591.yaml b/docs/changelog/106591.yaml deleted file mode 100644 index 6a7814cb9cede..0000000000000 --- a/docs/changelog/106591.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 106591 -summary: Make dense vector field type updatable -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/106820.yaml b/docs/changelog/106820.yaml deleted file mode 100644 index d854e3984c13d..0000000000000 --- a/docs/changelog/106820.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 106820 -summary: Add a capabilities API to check node and cluster capabilities -area: Infra/REST API -type: feature -issues: [] diff --git a/docs/changelog/107081.yaml b/docs/changelog/107081.yaml deleted file mode 100644 index 2acd2f919b476..0000000000000 --- a/docs/changelog/107081.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107081 -summary: Implement synthetic source support for range fields -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/107088.yaml b/docs/changelog/107088.yaml deleted file mode 100644 index 01a926f185eea..0000000000000 --- a/docs/changelog/107088.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107088 -summary: Introduce role description field -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/107191.yaml b/docs/changelog/107191.yaml deleted file mode 100644 index 5ef6297c0f3f1..0000000000000 --- a/docs/changelog/107191.yaml +++ /dev/null @@ -1,17 +0,0 @@ -pr: 107191 -summary: Stricter failure handling in multi-repo get-snapshots request handling -area: Snapshot/Restore -type: bug -issues: [] -highlight: - title: Stricter failure handling in multi-repo get-snapshots request handling - body: | - If a multi-repo get-snapshots request encounters a failure in one of the - targeted repositories then earlier versions of Elasticsearch would proceed - as if the faulty repository did not exist, except for a per-repository - failure report in a separate section of the response body. This makes it - impossible to paginate the results properly in the presence of failures. In - versions 8.15.0 and later this API's failure handling behaviour has been - made stricter, reporting an overall failure if any targeted repository's - contents cannot be listed. - notable: true diff --git a/docs/changelog/107216.yaml b/docs/changelog/107216.yaml deleted file mode 100644 index 7144eedf9bea4..0000000000000 --- a/docs/changelog/107216.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107216 -summary: Add per-field KNN vector format to Index Segments API -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/107240.yaml b/docs/changelog/107240.yaml deleted file mode 100644 index baf4c222a9a27..0000000000000 --- a/docs/changelog/107240.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107240 -summary: Include doc size info in ingest stats -area: Ingest Node -type: enhancement -issues: - - 106386 diff --git a/docs/changelog/107244.yaml b/docs/changelog/107244.yaml deleted file mode 100644 index f805796674f93..0000000000000 --- a/docs/changelog/107244.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107244 -summary: Support effective watermark thresholds in node stats API -area: Allocation -type: enhancement -issues: [106676] diff --git a/docs/changelog/107279.yaml b/docs/changelog/107279.yaml deleted file mode 100644 index a2940ecc9ba2d..0000000000000 --- a/docs/changelog/107279.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107279 -summary: Introduce _transform/_node_stats API -area: Transform -type: feature -issues: [] diff --git a/docs/changelog/107409.yaml b/docs/changelog/107409.yaml deleted file mode 100644 index 6f2350239772f..0000000000000 --- a/docs/changelog/107409.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107409 -summary: "ESQL: Introduce a casting operator, `::`" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/107410.yaml b/docs/changelog/107410.yaml deleted file mode 100644 index 5026e88cfa762..0000000000000 --- a/docs/changelog/107410.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107410 -summary: Cluster-state based Security role mapper -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/107415.yaml b/docs/changelog/107415.yaml deleted file mode 100644 index 8877d0426c60d..0000000000000 --- a/docs/changelog/107415.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107415 -summary: Fix `DecayFunctions'` `toString` -area: Search -type: bug -issues: - - 100870 diff --git a/docs/changelog/107426.yaml b/docs/changelog/107426.yaml deleted file mode 100644 index 2feed3df56108..0000000000000 --- a/docs/changelog/107426.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107426 -summary: Support wait indefinitely for search tasks to complete on node shutdown -area: Infra/Node Lifecycle -type: bug -issues: [] diff --git a/docs/changelog/107435.yaml b/docs/changelog/107435.yaml deleted file mode 100644 index ae5d2215419c4..0000000000000 --- a/docs/changelog/107435.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107435 -summary: '`NoSuchRemoteClusterException` should not be thrown when a remote is configured' -area: Network -type: bug -issues: - - 107381 diff --git a/docs/changelog/107493.yaml b/docs/changelog/107493.yaml deleted file mode 100644 index dfd45e1493c95..0000000000000 --- a/docs/changelog/107493.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107493 -summary: Remote cluster - API key security model - cluster privileges -area: Security -type: enhancement -issues: [] diff --git a/docs/changelog/107545.yaml b/docs/changelog/107545.yaml deleted file mode 100644 index ad457cc5a533f..0000000000000 --- a/docs/changelog/107545.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107545 -summary: "ESQL: Union Types Support" -area: ES|QL -type: enhancement -issues: - - 100603 diff --git a/docs/changelog/107549.yaml b/docs/changelog/107549.yaml deleted file mode 100644 index 36250cf65b4d9..0000000000000 --- a/docs/changelog/107549.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107549 -summary: Add synthetic source support for binary fields -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/107567.yaml b/docs/changelog/107567.yaml deleted file mode 100644 index 558b5b570b1fb..0000000000000 --- a/docs/changelog/107567.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107567 -summary: Add ignored field values to synthetic source -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/107579.yaml b/docs/changelog/107579.yaml deleted file mode 100644 index fdee59424b8de..0000000000000 --- a/docs/changelog/107579.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107579 -summary: Adding `hits_time_in_millis` and `misses_time_in_millis` to enrich cache - stats -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/107593.yaml b/docs/changelog/107593.yaml deleted file mode 100644 index 2e3d2cbc80119..0000000000000 --- a/docs/changelog/107593.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107593 -summary: Add auto-sharding APM metrics -area: Infra/Metrics -type: enhancement -issues: [] diff --git a/docs/changelog/107640.yaml b/docs/changelog/107640.yaml deleted file mode 100644 index 9871943481f20..0000000000000 --- a/docs/changelog/107640.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107640 -summary: "Unified Highlighter to support matched_fields " -area: Highlighting -type: enhancement -issues: - - 5172 diff --git a/docs/changelog/107645.yaml b/docs/changelog/107645.yaml deleted file mode 100644 index 93fc0f2a89b3a..0000000000000 --- a/docs/changelog/107645.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 107645 -summary: Add `_name` support for top level `knn` clauses -area: Search -type: enhancement -issues: - - 106254 - - 107448 diff --git a/docs/changelog/107647.yaml b/docs/changelog/107647.yaml deleted file mode 100644 index 97d98a7c91079..0000000000000 --- a/docs/changelog/107647.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107647 -summary: Adding human readable times to geoip stats -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/107663.yaml b/docs/changelog/107663.yaml deleted file mode 100644 index a7c3dc185425a..0000000000000 --- a/docs/changelog/107663.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107663 -summary: Optimize `GeoBounds` and `GeoCentroid` aggregations for single value fields -area: Geo -type: enhancement -issues: [] diff --git a/docs/changelog/107675.yaml b/docs/changelog/107675.yaml deleted file mode 100644 index b1d51cd3f8538..0000000000000 --- a/docs/changelog/107675.yaml +++ /dev/null @@ -1,17 +0,0 @@ -pr: 107675 -summary: Interpret `?timeout=-1` as infinite ack timeout -area: Cluster Coordination -type: breaking -issues: [] -breaking: - title: Interpret `?timeout=-1` as infinite ack timeout - area: REST API - details: | - Today {es} accepts the parameter `?timeout=-1` in many APIs, but interprets - this to mean the same as `?timeout=0`. From 8.15 onwards `?timeout=-1` will - mean to wait indefinitely, aligning the behaviour of this parameter with - other similar parameters such as `?master_timeout`. - impact: | - Use `?timeout=0` to force relevant operations to time out immediately - instead of `?timeout=-1` - notable: false diff --git a/docs/changelog/107676.yaml b/docs/changelog/107676.yaml deleted file mode 100644 index b14bc29e66efd..0000000000000 --- a/docs/changelog/107676.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107676 -summary: Add model download progress to the download task status -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/107706.yaml b/docs/changelog/107706.yaml deleted file mode 100644 index 76b7f662bf0e0..0000000000000 --- a/docs/changelog/107706.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107706 -summary: Add rate limiting support for the Inference API -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/107735.yaml b/docs/changelog/107735.yaml deleted file mode 100644 index 372cb59ba8b1f..0000000000000 --- a/docs/changelog/107735.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107735 -summary: Implement synthetic source support for annotated text field -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/107739.yaml b/docs/changelog/107739.yaml deleted file mode 100644 index c55a0e332b4f6..0000000000000 --- a/docs/changelog/107739.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107739 -summary: Binary field enables doc values by default for index mode with synthe… -area: Mapping -type: enhancement -issues: - - 107554 diff --git a/docs/changelog/107764.yaml b/docs/changelog/107764.yaml deleted file mode 100644 index 3f83efc789014..0000000000000 --- a/docs/changelog/107764.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107764 -summary: Increase size of big arrays only when there is an actual value in the aggregators -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107779.yaml b/docs/changelog/107779.yaml deleted file mode 100644 index a41c19a2329e0..0000000000000 --- a/docs/changelog/107779.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107779 -summary: Allow rescorer with field collapsing -area: Search -type: enhancement -issues: - - 27243 \ No newline at end of file diff --git a/docs/changelog/107792.yaml b/docs/changelog/107792.yaml deleted file mode 100644 index bd9730d49d5d6..0000000000000 --- a/docs/changelog/107792.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107792 -summary: Halt Indexer on Stop/Abort API -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/107813.yaml b/docs/changelog/107813.yaml deleted file mode 100644 index 1cbb518a8be5b..0000000000000 --- a/docs/changelog/107813.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107813 -summary: Increase size of big arrays only when there is an actual value in the aggregators - (Analytics module) -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107827.yaml b/docs/changelog/107827.yaml deleted file mode 100644 index 7cf217567b745..0000000000000 --- a/docs/changelog/107827.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107827 -summary: Add permission to secure access to certain config files -area: Security -type: bug -issues: [] diff --git a/docs/changelog/107832.yaml b/docs/changelog/107832.yaml deleted file mode 100644 index 491c491736005..0000000000000 --- a/docs/changelog/107832.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107832 -summary: Optimise few metric aggregations for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107862.yaml b/docs/changelog/107862.yaml deleted file mode 100644 index 77f7a8c9fb02a..0000000000000 --- a/docs/changelog/107862.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107862 -summary: Fix serialization of put-shutdown request -area: Infra/Node Lifecycle -type: bug -issues: - - 107857 diff --git a/docs/changelog/107876.yaml b/docs/changelog/107876.yaml deleted file mode 100644 index 21624cacf7e1d..0000000000000 --- a/docs/changelog/107876.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107876 -summary: "ESQL: Add aggregates node level reduction" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/107877.yaml b/docs/changelog/107877.yaml deleted file mode 100644 index cf458b3aa3a42..0000000000000 --- a/docs/changelog/107877.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107877 -summary: Support metrics counter types in ESQL -area: "ES|QL" -type: enhancement -issues: [] diff --git a/docs/changelog/107886.yaml b/docs/changelog/107886.yaml deleted file mode 100644 index a328bc2a2a208..0000000000000 --- a/docs/changelog/107886.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107886 -summary: Cluster state role mapper file settings service -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/107892.yaml b/docs/changelog/107892.yaml deleted file mode 100644 index 5fd5404c48d02..0000000000000 --- a/docs/changelog/107892.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107892 -summary: Optimise cardinality aggregations for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107893.yaml b/docs/changelog/107893.yaml deleted file mode 100644 index 61f0f4d76e679..0000000000000 --- a/docs/changelog/107893.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107893 -summary: Optimise histogram aggregations for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107897.yaml b/docs/changelog/107897.yaml deleted file mode 100644 index e4a2a5270475d..0000000000000 --- a/docs/changelog/107897.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107897 -summary: Optimise composite aggregations for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107917.yaml b/docs/changelog/107917.yaml deleted file mode 100644 index 18125bf46f4b7..0000000000000 --- a/docs/changelog/107917.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107917 -summary: Exit gracefully when deleted -area: Transform -type: bug -issues: - - 107266 diff --git a/docs/changelog/107922.yaml b/docs/changelog/107922.yaml deleted file mode 100644 index e28d0f6262af4..0000000000000 --- a/docs/changelog/107922.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107922 -summary: Feature/annotated text store defaults -area: Mapping -type: enhancement -issues: - - 107734 diff --git a/docs/changelog/107930.yaml b/docs/changelog/107930.yaml deleted file mode 100644 index 90af5c55b8604..0000000000000 --- a/docs/changelog/107930.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107930 -summary: Optimise terms aggregations for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107937.yaml b/docs/changelog/107937.yaml deleted file mode 100644 index 5938c8e8b6602..0000000000000 --- a/docs/changelog/107937.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107937 -summary: Optimise multiterms aggregation for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/107947.yaml b/docs/changelog/107947.yaml deleted file mode 100644 index 637ac3c005779..0000000000000 --- a/docs/changelog/107947.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107947 -summary: "ESQL: Fix equals `hashCode` for functions" -area: ES|QL -type: bug -issues: - - 104393 diff --git a/docs/changelog/107967.yaml b/docs/changelog/107967.yaml deleted file mode 100644 index 159370e44f236..0000000000000 --- a/docs/changelog/107967.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107967 -summary: Sort time series indices by time range in `GetDataStreams` API -area: TSDB -type: bug -issues: - - 102088 diff --git a/docs/changelog/107972.yaml b/docs/changelog/107972.yaml deleted file mode 100644 index 3ec83d6a56954..0000000000000 --- a/docs/changelog/107972.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107972 -summary: Require question to be non-null in `QuestionAnsweringConfig` -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/107977.yaml b/docs/changelog/107977.yaml deleted file mode 100644 index fdbbb57d7e48f..0000000000000 --- a/docs/changelog/107977.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107977 -summary: Fix off by one error when handling null values in range fields -area: Mapping -type: bug -issues: - - 107282 diff --git a/docs/changelog/107978.yaml b/docs/changelog/107978.yaml deleted file mode 100644 index 50115df9ee092..0000000000000 --- a/docs/changelog/107978.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107978 -summary: Drop shards close timeout when stopping node. -area: Engine -type: enhancement -issues: - - 107938 diff --git a/docs/changelog/107987.yaml b/docs/changelog/107987.yaml deleted file mode 100644 index e8afebde0b190..0000000000000 --- a/docs/changelog/107987.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107987 -summary: "ESQL: Implement LOOKUP, an \"inline\" enrich" -area: ES|QL -type: enhancement -issues: - - 107306 diff --git a/docs/changelog/107990.yaml b/docs/changelog/107990.yaml deleted file mode 100644 index 80cb96aca4426..0000000000000 --- a/docs/changelog/107990.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107990 -summary: Optimise `time_series` aggregation for single value fields -area: TSDB -type: enhancement -issues: [] diff --git a/docs/changelog/108016.yaml b/docs/changelog/108016.yaml deleted file mode 100644 index 0aa3f86a6f859..0000000000000 --- a/docs/changelog/108016.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108016 -summary: Optimise `BinaryRangeAggregator` for single value fields -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/108019.yaml b/docs/changelog/108019.yaml deleted file mode 100644 index 69e8e9fd371f8..0000000000000 --- a/docs/changelog/108019.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108019 -summary: Ignore additional cpu.stat fields -area: Infra/Core -type: bug -issues: - - 107983 diff --git a/docs/changelog/108051.yaml b/docs/changelog/108051.yaml deleted file mode 100644 index a47e1192c6090..0000000000000 --- a/docs/changelog/108051.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108051 -summary: Track synthetic source for disabled objects -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/108065.yaml b/docs/changelog/108065.yaml deleted file mode 100644 index 2ec93bf6e6295..0000000000000 --- a/docs/changelog/108065.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108065 -summary: '`DenseVectorFieldMapper` fixed typo' -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/108070.yaml b/docs/changelog/108070.yaml deleted file mode 100644 index cde191aa50804..0000000000000 --- a/docs/changelog/108070.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108070 -summary: Redirect `VersionConflict` to reset code -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/108088.yaml b/docs/changelog/108088.yaml deleted file mode 100644 index 95c58f6dc19f1..0000000000000 --- a/docs/changelog/108088.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108088 -summary: Add a SIMD (AVX2) optimised vector distance function for int7 on x64 -area: "Search" -type: enhancement -issues: [] diff --git a/docs/changelog/108089.yaml b/docs/changelog/108089.yaml deleted file mode 100644 index 02fb6349185a6..0000000000000 --- a/docs/changelog/108089.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108089 -summary: "ES|QL: limit query depth to 500 levels" -area: ES|QL -type: bug -issues: - - 107752 diff --git a/docs/changelog/108106.yaml b/docs/changelog/108106.yaml deleted file mode 100644 index e9dd438e620c4..0000000000000 --- a/docs/changelog/108106.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108106 -summary: Simulate should succeed if `ignore_missing_pipeline` -area: Ingest Node -type: bug -issues: - - 107314 diff --git a/docs/changelog/108118.yaml b/docs/changelog/108118.yaml deleted file mode 100644 index b9b0f1c1406e0..0000000000000 --- a/docs/changelog/108118.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108118 -summary: Optimize for single value in ordinals grouping -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108122.yaml b/docs/changelog/108122.yaml deleted file mode 100644 index 981ab39b9dad8..0000000000000 --- a/docs/changelog/108122.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108122 -summary: Correct query profiling for conjunctions -area: Search -type: bug -issues: - - 108116 diff --git a/docs/changelog/108130.yaml b/docs/changelog/108130.yaml deleted file mode 100644 index 5b431bdb0cc1b..0000000000000 --- a/docs/changelog/108130.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108130 -summary: Optimise frequent item sets aggregation for single value fields -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/108131.yaml b/docs/changelog/108131.yaml deleted file mode 100644 index 7a4286c1e44a0..0000000000000 --- a/docs/changelog/108131.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108131 -summary: "Inference Processor: skip inference when all fields are missing" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/108144.yaml b/docs/changelog/108144.yaml deleted file mode 100644 index 6ff5b1d600d0e..0000000000000 --- a/docs/changelog/108144.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108144 -summary: Bump Tika dependency to 2.9.2 -area: Ingest Node -type: upgrade -issues: [] diff --git a/docs/changelog/108145.yaml b/docs/changelog/108145.yaml deleted file mode 100644 index b8c9428c1e3a8..0000000000000 --- a/docs/changelog/108145.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108145 -summary: Async close of `IndexShard` -area: Engine -type: bug -issues: [] diff --git a/docs/changelog/108146.yaml b/docs/changelog/108146.yaml deleted file mode 100644 index 2a4f917134090..0000000000000 --- a/docs/changelog/108146.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108146 -summary: Allow deletion of the ELSER inference service when reference in ingest -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/108155.yaml b/docs/changelog/108155.yaml deleted file mode 100644 index 57db86b4005b9..0000000000000 --- a/docs/changelog/108155.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108155 -summary: Upgrade to Netty 4.1.109 -area: Network -type: upgrade -issues: [] diff --git a/docs/changelog/108161.yaml b/docs/changelog/108161.yaml deleted file mode 100644 index 73fa41e2089d3..0000000000000 --- a/docs/changelog/108161.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108161 -summary: Refactor TextEmbeddingResults to use primitives rather than objects -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/108165.yaml b/docs/changelog/108165.yaml deleted file mode 100644 index b88b0f5e217dd..0000000000000 --- a/docs/changelog/108165.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108165 -summary: Add `BlockHash` for 3 `BytesRefs` -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108171.yaml b/docs/changelog/108171.yaml deleted file mode 100644 index 1ec17bb3e411d..0000000000000 --- a/docs/changelog/108171.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108171 -summary: "add Elastic-internal stable bridge api for use by Logstash" -area: Infra/Core -type: enhancement -issues: [] diff --git a/docs/changelog/108222.yaml b/docs/changelog/108222.yaml deleted file mode 100644 index 701b853441e32..0000000000000 --- a/docs/changelog/108222.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108222 -summary: Add generic fallback implementation for synthetic source -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/108223.yaml b/docs/changelog/108223.yaml deleted file mode 100644 index ba8756a8f9c68..0000000000000 --- a/docs/changelog/108223.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108223 -summary: Upgrade bouncy castle (non-fips) to 1.78.1 -area: Security -type: upgrade -issues: [] diff --git a/docs/changelog/108227.yaml b/docs/changelog/108227.yaml deleted file mode 100644 index 79f69bc4aaff6..0000000000000 --- a/docs/changelog/108227.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108227 -summary: "Apm-data: improve indexing resilience" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/108254.yaml b/docs/changelog/108254.yaml deleted file mode 100644 index 3bf08e8b8f5fc..0000000000000 --- a/docs/changelog/108254.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108254 -summary: Add `sparse_vector` query -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/108266.yaml b/docs/changelog/108266.yaml deleted file mode 100644 index 5a189cfcdc258..0000000000000 --- a/docs/changelog/108266.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108266 -summary: Log details of non-green indicators in `HealthPeriodicLogger` -area: Health -type: enhancement -issues: [] diff --git a/docs/changelog/108300.yaml b/docs/changelog/108300.yaml deleted file mode 100644 index c4d6e468113a4..0000000000000 --- a/docs/changelog/108300.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108300 -summary: "ESQL: Add more time span units" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108306.yaml b/docs/changelog/108306.yaml deleted file mode 100644 index 7a104ce880f43..0000000000000 --- a/docs/changelog/108306.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108306 -summary: Enable inter-segment concurrency for low cardinality numeric terms aggs -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/108333.yaml b/docs/changelog/108333.yaml deleted file mode 100644 index c3152500ce1b2..0000000000000 --- a/docs/changelog/108333.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108333 -summary: Allow `read_slm` to call GET /_slm/status -area: ILM+SLM -type: bug -issues: [] diff --git a/docs/changelog/108340.yaml b/docs/changelog/108340.yaml deleted file mode 100644 index fb2ea72c0a0f5..0000000000000 --- a/docs/changelog/108340.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108340 -summary: "Apm-data: increase version for templates" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/108349.yaml b/docs/changelog/108349.yaml deleted file mode 100644 index 6d9ea3d658dca..0000000000000 --- a/docs/changelog/108349.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108349 -summary: "Ecs@mappings: reduce scope for `ecs_geo_point`" -area: Data streams -type: bug -issues: - - 108338 diff --git a/docs/changelog/108360.yaml b/docs/changelog/108360.yaml deleted file mode 100644 index 087dd2649c6aa..0000000000000 --- a/docs/changelog/108360.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108360 -summary: "ESQL: Fix variable shadowing when pushing down past Project" -area: ES|QL -type: bug -issues: - - 108008 diff --git a/docs/changelog/108379.yaml b/docs/changelog/108379.yaml deleted file mode 100644 index 312856a5db33d..0000000000000 --- a/docs/changelog/108379.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108379 -summary: Create a new `NodeRequest` for every `NodesDataTiersUsageTransport` use -area: Indices APIs -type: bug -issues: [] diff --git a/docs/changelog/108394.yaml b/docs/changelog/108394.yaml deleted file mode 100644 index 58f48fa548c6e..0000000000000 --- a/docs/changelog/108394.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108394 -summary: Handle `IndexNotFoundException` -area: Transform -type: bug -issues: - - 107263 diff --git a/docs/changelog/108395.yaml b/docs/changelog/108395.yaml deleted file mode 100644 index c33cf169a99fa..0000000000000 --- a/docs/changelog/108395.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108395 -summary: "ESQL: change from quoting from backtick to quote" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/108396.yaml b/docs/changelog/108396.yaml deleted file mode 100644 index 63937646b755c..0000000000000 --- a/docs/changelog/108396.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108396 -summary: "Apm-data: improve default pipeline performance" -area: Data streams -type: enhancement -issues: - - 108290 diff --git a/docs/changelog/108409.yaml b/docs/changelog/108409.yaml deleted file mode 100644 index 6cff86cf93930..0000000000000 --- a/docs/changelog/108409.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108409 -summary: Support multiple associated groups for TopN -area: Application -type: enhancement -issues: - - 108018 diff --git a/docs/changelog/108410.yaml b/docs/changelog/108410.yaml deleted file mode 100644 index 5fd831231a3be..0000000000000 --- a/docs/changelog/108410.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108410 -summary: GeoIP tasks should wait longer for master -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/108417.yaml b/docs/changelog/108417.yaml deleted file mode 100644 index bb650922f1be5..0000000000000 --- a/docs/changelog/108417.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108417 -summary: Track source for arrays of objects -area: Mapping -type: enhancement -issues: - - 90708 diff --git a/docs/changelog/108421.yaml b/docs/changelog/108421.yaml deleted file mode 100644 index 1f077a4a2cb7c..0000000000000 --- a/docs/changelog/108421.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108421 -summary: "[ES|QL] Support Named and Positional Parameters in `EsqlQueryRequest`" -area: ES|QL -type: enhancement -issues: - - 107029 diff --git a/docs/changelog/108429.yaml b/docs/changelog/108429.yaml deleted file mode 100644 index 562454a0de256..0000000000000 --- a/docs/changelog/108429.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108429 -summary: Fix `ClassCastException` in Significant Terms -area: Aggregations -type: bug -issues: - - 108427 diff --git a/docs/changelog/108444.yaml b/docs/changelog/108444.yaml deleted file mode 100644 index c946ab24f939a..0000000000000 --- a/docs/changelog/108444.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108444 -summary: "Apm-data: ignore malformed fields, and too many dynamic fields" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/108452.yaml b/docs/changelog/108452.yaml deleted file mode 100644 index fdf531602c806..0000000000000 --- a/docs/changelog/108452.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108452 -summary: Add the rerank task to the Elasticsearch internal inference service -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/108455.yaml b/docs/changelog/108455.yaml deleted file mode 100644 index 8397af7b07cf1..0000000000000 --- a/docs/changelog/108455.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108455 -summary: "[ES|QL] Convert string to datetime when the other size of an arithmetic\ - \ operator is `date_period` or `time_duration`" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108459.yaml b/docs/changelog/108459.yaml deleted file mode 100644 index 5e05797f284be..0000000000000 --- a/docs/changelog/108459.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108459 -summary: Do not use global ordinals strategy if the leaf reader context cannot be - obtained -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/108472.yaml b/docs/changelog/108472.yaml deleted file mode 100644 index 82481e4edec3a..0000000000000 --- a/docs/changelog/108472.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108472 -summary: Add support for Azure AI Studio embeddings and completions to the inference service. -area: Machine Learning -type: feature -issues: [] diff --git a/docs/changelog/108517.yaml b/docs/changelog/108517.yaml deleted file mode 100644 index 359c8302fdf6c..0000000000000 --- a/docs/changelog/108517.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108517 -summary: Forward `indexServiceSafe` exception to listener -area: Transform -type: bug -issues: - - 108418 diff --git a/docs/changelog/108521.yaml b/docs/changelog/108521.yaml deleted file mode 100644 index adc7c11a4decd..0000000000000 --- a/docs/changelog/108521.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108521 -summary: Adding override for lintian false positive on `libvec.so` -area: "Packaging" -type: bug -issues: - - 108514 diff --git a/docs/changelog/108522.yaml b/docs/changelog/108522.yaml deleted file mode 100644 index 5bc064d7995e9..0000000000000 --- a/docs/changelog/108522.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108522 -summary: Ensure we return non-negative scores when scoring scalar dot-products -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/108537.yaml b/docs/changelog/108537.yaml deleted file mode 100644 index 1c0228a71d449..0000000000000 --- a/docs/changelog/108537.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108537 -summary: Limit the value in prefix query -area: Search -type: enhancement -issues: - - 108486 diff --git a/docs/changelog/108538.yaml b/docs/changelog/108538.yaml deleted file mode 100644 index 10ae49f0c1670..0000000000000 --- a/docs/changelog/108538.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108538 -summary: Adding RankFeature search phase implementation -area: Search -type: feature -issues: [] diff --git a/docs/changelog/108574.yaml b/docs/changelog/108574.yaml deleted file mode 100644 index b3c957721e01e..0000000000000 --- a/docs/changelog/108574.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108574 -summary: "[ESQL] CBRT function" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108602.yaml b/docs/changelog/108602.yaml deleted file mode 100644 index d544c89980123..0000000000000 --- a/docs/changelog/108602.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108602 -summary: "[Inference API] Extract optional long instead of integer in `RateLimitSettings#of`" -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/108606.yaml b/docs/changelog/108606.yaml deleted file mode 100644 index 04780bff58800..0000000000000 --- a/docs/changelog/108606.yaml +++ /dev/null @@ -1,14 +0,0 @@ -pr: 108606 -summary: "Extend ISO8601 datetime parser to specify forbidden fields, allowing it to be used\ - \ on more formats" -area: Infra/Core -type: enhancement -issues: [] -highlight: - title: New custom parser for more ISO-8601 date formats - body: |- - Following on from #106486, this extends the custom ISO-8601 datetime parser to cover the `strict_year`, - `strict_year_month`, `strict_date_time`, `strict_date_time_no_millis`, `strict_date_hour_minute_second`, - `strict_date_hour_minute_second_millis`, and `strict_date_hour_minute_second_fraction` date formats. - As before, the parser will use the existing java.time parser if there are parsing issues, and the - `es.datetime.java_time_parsers=true` JVM property will force the use of the old parsers regardless. diff --git a/docs/changelog/108607.yaml b/docs/changelog/108607.yaml deleted file mode 100644 index 9ad4cf91e67b9..0000000000000 --- a/docs/changelog/108607.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108607 -summary: Specify parse index when error occurs on multiple datetime parses -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/108612.yaml b/docs/changelog/108612.yaml deleted file mode 100644 index 7a3dfa2b7ba44..0000000000000 --- a/docs/changelog/108612.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108612 -summary: "[Connector API] Change `UpdateConnectorFiltering` API to have better defaults" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/108624.yaml b/docs/changelog/108624.yaml deleted file mode 100644 index 0da1fd2902c03..0000000000000 --- a/docs/changelog/108624.yaml +++ /dev/null @@ -1,12 +0,0 @@ -pr: 108624 -summary: Disallow new rollup jobs in clusters with no rollup usage -area: Rollup -type: breaking -issues: - - 108381 -breaking: - title: Disallow new rollup jobs in clusters with no rollup usage - area: Rollup - details: The put rollup API will fail with an error when a rollup job is created in a cluster with no rollup usage - impact: Clusters with no rollup usage (either no rollup job or index) can not create new rollup jobs - notable: true diff --git a/docs/changelog/108639.yaml b/docs/changelog/108639.yaml deleted file mode 100644 index e4964cbeb0285..0000000000000 --- a/docs/changelog/108639.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108639 -summary: Add support for the 'Domain' database to the geoip processor -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/108643.yaml b/docs/changelog/108643.yaml deleted file mode 100644 index f71a943673326..0000000000000 --- a/docs/changelog/108643.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108643 -summary: Use `scheduleUnlessShuttingDown` in `LeaderChecker` -area: Cluster Coordination -type: bug -issues: - - 108642 diff --git a/docs/changelog/108651.yaml b/docs/changelog/108651.yaml deleted file mode 100644 index 227c464909d50..0000000000000 --- a/docs/changelog/108651.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108651 -summary: Add support for the 'ISP' database to the geoip processor -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/108672.yaml b/docs/changelog/108672.yaml deleted file mode 100644 index e1261fcf6f232..0000000000000 --- a/docs/changelog/108672.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108672 -summary: Add bounds checking to parsing ISO8601 timezone offset values -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/108679.yaml b/docs/changelog/108679.yaml deleted file mode 100644 index 62cd82a52c5bb..0000000000000 --- a/docs/changelog/108679.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108679 -summary: Suppress deprecation warnings from ingest pipelines when deleting trained model -area: Machine Learning -type: bug -issues: - - 105004 diff --git a/docs/changelog/108682.yaml b/docs/changelog/108682.yaml deleted file mode 100644 index bd566acab8306..0000000000000 --- a/docs/changelog/108682.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108682 -summary: Adding support for explain in rrf -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/108683.yaml b/docs/changelog/108683.yaml deleted file mode 100644 index b9e7df5fefc18..0000000000000 --- a/docs/changelog/108683.yaml +++ /dev/null @@ -1,14 +0,0 @@ -pr: 108683 -summary: Add support for the 'Connection Type' database to the geoip processor -area: Ingest Node -type: enhancement -issues: [] -highlight: - title: "Preview: Support for the 'Connection Type, 'Domain', and 'ISP' databases in the geoip processor" - body: |- - As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial - https://dev.maxmind.com/geoip/docs/databases/connection-type[GeoIP2 'Connection Type'], - https://dev.maxmind.com/geoip/docs/databases/domain[GeoIP2 'Domain'], - and - https://dev.maxmind.com/geoip/docs/databases/isp[GeoIP2 'ISP'] - databases from MaxMind. diff --git a/docs/changelog/108684.yaml b/docs/changelog/108684.yaml deleted file mode 100644 index 91684d2998be6..0000000000000 --- a/docs/changelog/108684.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108684 -summary: Check if `CsvTests` required capabilities exist -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108687.yaml b/docs/changelog/108687.yaml deleted file mode 100644 index 771516d551567..0000000000000 --- a/docs/changelog/108687.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108687 -summary: Adding `user_type` support for the enterprise database for the geoip processor -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/108693.yaml b/docs/changelog/108693.yaml deleted file mode 100644 index ee701e0f57736..0000000000000 --- a/docs/changelog/108693.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108693 -summary: Test pipeline run after reroute -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/108705.yaml b/docs/changelog/108705.yaml deleted file mode 100644 index fd08734831018..0000000000000 --- a/docs/changelog/108705.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108705 -summary: Associate restore snapshot task to parent mount task -area: Distributed -type: bug -issues: - - 105830 diff --git a/docs/changelog/108713.yaml b/docs/changelog/108713.yaml deleted file mode 100644 index d6b1ddabd6c1e..0000000000000 --- a/docs/changelog/108713.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108713 -summary: Rewrite away type converting functions that do not convert types -area: ES|QL -type: enhancement -issues: - - 107716 diff --git a/docs/changelog/108726.yaml b/docs/changelog/108726.yaml deleted file mode 100644 index 2e800a45e6975..0000000000000 --- a/docs/changelog/108726.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108726 -summary: Allow RA metrics to be reported upon parsing completed or accumulated -area: Infra/Metrics -type: enhancement -issues: [] diff --git a/docs/changelog/108733.yaml b/docs/changelog/108733.yaml deleted file mode 100644 index 76a969219ea4c..0000000000000 --- a/docs/changelog/108733.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108733 -summary: Query Roles API -area: Security -type: feature -issues: [] diff --git a/docs/changelog/108746.yaml b/docs/changelog/108746.yaml deleted file mode 100644 index 93ed917f3b56e..0000000000000 --- a/docs/changelog/108746.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108746 -summary: Support synthetic source for `aggregate_metric_double` when ignore_malf… -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/108759.yaml b/docs/changelog/108759.yaml deleted file mode 100644 index dfc2b30fe6c57..0000000000000 --- a/docs/changelog/108759.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108759 -summary: Expose `?master_timeout` in autoscaling APIs -area: Autoscaling -type: bug -issues: [] diff --git a/docs/changelog/108761.yaml b/docs/changelog/108761.yaml deleted file mode 100644 index 92aa67ebe0bfe..0000000000000 --- a/docs/changelog/108761.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108761 -summary: Add some missing timeout params to REST API specs -area: Infra/REST API -type: bug -issues: [] diff --git a/docs/changelog/108764.yaml b/docs/changelog/108764.yaml deleted file mode 100644 index 94de27eb52c9b..0000000000000 --- a/docs/changelog/108764.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108764 -summary: ST_DISTANCE Function -area: ES|QL -type: enhancement -issues: - - 108212 diff --git a/docs/changelog/108780.yaml b/docs/changelog/108780.yaml deleted file mode 100644 index 40e66326e6b9b..0000000000000 --- a/docs/changelog/108780.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108780 -summary: Add `continent_code` support to the geoip processor -area: Ingest Node -type: enhancement -issues: - - 85820 diff --git a/docs/changelog/108786.yaml b/docs/changelog/108786.yaml deleted file mode 100644 index 1c07a3ceac900..0000000000000 --- a/docs/changelog/108786.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108786 -summary: Make ingest byte stat names more descriptive -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/108793.yaml b/docs/changelog/108793.yaml deleted file mode 100644 index 87668c8ee009b..0000000000000 --- a/docs/changelog/108793.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108793 -summary: Add `SparseVectorStats` -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/108796.yaml b/docs/changelog/108796.yaml deleted file mode 100644 index 808247cf347d9..0000000000000 --- a/docs/changelog/108796.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108796 -summary: Return ingest byte stats even when 0-valued -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/108814.yaml b/docs/changelog/108814.yaml deleted file mode 100644 index 94298838c372e..0000000000000 --- a/docs/changelog/108814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108814 -summary: Deserialize publish requests on generic thread-pool -area: Cluster Coordination -type: bug -issues: - - 106352 diff --git a/docs/changelog/108818.yaml b/docs/changelog/108818.yaml deleted file mode 100644 index ed60fb5f64abd..0000000000000 --- a/docs/changelog/108818.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108818 -summary: Store source for nested objects -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/108820.yaml b/docs/changelog/108820.yaml deleted file mode 100644 index 55045ffce3dfa..0000000000000 --- a/docs/changelog/108820.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108820 -summary: Allow `LuceneSourceOperator` to early terminate -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/108822.yaml b/docs/changelog/108822.yaml deleted file mode 100644 index 8cec4da5dbc7f..0000000000000 --- a/docs/changelog/108822.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108822 -summary: Update ASM to 9.7 for plugin scanner -area: Infra/Plugins -type: upgrade -issues: - - 108776 diff --git a/docs/changelog/108831.yaml b/docs/changelog/108831.yaml deleted file mode 100644 index 496bc0108f9d2..0000000000000 --- a/docs/changelog/108831.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108831 -summary: Rename rule query and add support for multiple rulesets -area: Application -type: enhancement -issues: [ ] diff --git a/docs/changelog/108849.yaml b/docs/changelog/108849.yaml deleted file mode 100644 index 7c503efe9187b..0000000000000 --- a/docs/changelog/108849.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108849 -summary: "[Osquery] Extend `kibana_system` role with an access to new `osquery_manager`\ - \ index" -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/108856.yaml b/docs/changelog/108856.yaml deleted file mode 100644 index 9b8f42248a442..0000000000000 --- a/docs/changelog/108856.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108856 -summary: Return noop instance `DocSizeObserver` for updates with scripts -area: Infra/Metrics -type: enhancement -issues: [] diff --git a/docs/changelog/108860.yaml b/docs/changelog/108860.yaml deleted file mode 100644 index 93aa8ce7c08ff..0000000000000 --- a/docs/changelog/108860.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108860 -summary: "Apm-data: enable plugin by default" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/108862.yaml b/docs/changelog/108862.yaml deleted file mode 100644 index ddba15f11e8f5..0000000000000 --- a/docs/changelog/108862.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108862 -summary: "Apm-data: set codec: best_compression for logs-apm.* data streams" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/108868.yaml b/docs/changelog/108868.yaml deleted file mode 100644 index d0643f056cce8..0000000000000 --- a/docs/changelog/108868.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108868 -summary: GA the update trained model action -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/108870.yaml b/docs/changelog/108870.yaml deleted file mode 100644 index 435eea9845f16..0000000000000 --- a/docs/changelog/108870.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108870 -summary: Adding score from `RankDoc` to `SearchHit` -area: Search -type: bug -issues: [] diff --git a/docs/changelog/108871.yaml b/docs/changelog/108871.yaml deleted file mode 100644 index 46bf8ca9d8404..0000000000000 --- a/docs/changelog/108871.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108871 -summary: "Reapply \"ESQL: Expose \"_ignored\" metadata field\"" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/108878.yaml b/docs/changelog/108878.yaml deleted file mode 100644 index 1a8127869a647..0000000000000 --- a/docs/changelog/108878.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108878 -summary: Support arrays in fallback synthetic source implementation -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/108881.yaml b/docs/changelog/108881.yaml deleted file mode 100644 index b6de1129cfa03..0000000000000 --- a/docs/changelog/108881.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108881 -summary: Add synthetic source support for `geo_shape` via fallback implementation -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/108885.yaml b/docs/changelog/108885.yaml deleted file mode 100644 index c66843e082e29..0000000000000 --- a/docs/changelog/108885.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108885 -summary: "Apm-data: increase priority above Fleet templates" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/108886.yaml b/docs/changelog/108886.yaml deleted file mode 100644 index 18df59e577713..0000000000000 --- a/docs/changelog/108886.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108886 -summary: Expose `?master_timeout` on get-shutdown API -area: Infra/Node Lifecycle -type: bug -issues: [] diff --git a/docs/changelog/108891.yaml b/docs/changelog/108891.yaml deleted file mode 100644 index 8282b616b34a9..0000000000000 --- a/docs/changelog/108891.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108891 -summary: Fix NPE during destination index creation -area: Transform -type: bug -issues: - - 108890 diff --git a/docs/changelog/108895.yaml b/docs/changelog/108895.yaml deleted file mode 100644 index 15293896b20c5..0000000000000 --- a/docs/changelog/108895.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108895 -summary: Add permission to secure access to certain config files specified by settings -area: "Security" -type: bug -issues: [] diff --git a/docs/changelog/108896.yaml b/docs/changelog/108896.yaml deleted file mode 100644 index c52f074b65605..0000000000000 --- a/docs/changelog/108896.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 108896 -summary: Introduce `logs` index mode as Tech Preview -area: Logs -type: feature -issues: - - 108896 diff --git a/docs/changelog/108911.yaml b/docs/changelog/108911.yaml deleted file mode 100644 index 8832e01f7426e..0000000000000 --- a/docs/changelog/108911.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108911 -summary: Store source for fields in objects with `dynamic` override -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/108942.yaml b/docs/changelog/108942.yaml deleted file mode 100644 index c58b06a92cee8..0000000000000 --- a/docs/changelog/108942.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108942 -summary: Fix NPE in trained model assignment updater -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/108947.yaml b/docs/changelog/108947.yaml deleted file mode 100644 index 8aa4293242985..0000000000000 --- a/docs/changelog/108947.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108947 -summary: Provide the `DocumentSizeReporter` with index mode -area: Infra/Metrics -type: enhancement -issues: [] diff --git a/docs/changelog/108999.yaml b/docs/changelog/108999.yaml deleted file mode 100644 index 089d765b4e2d0..0000000000000 --- a/docs/changelog/108999.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108999 -summary: Use default translog durability on AD results index -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/109007.yaml b/docs/changelog/109007.yaml deleted file mode 100644 index c828db64220fb..0000000000000 --- a/docs/changelog/109007.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109007 -summary: Multivalue Sparse Vector Support -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/109025.yaml b/docs/changelog/109025.yaml deleted file mode 100644 index 38d19cab13d30..0000000000000 --- a/docs/changelog/109025.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109025 -summary: Introduce a setting controlling the activation of the `logs` index mode in logs@settings -area: Logs -type: feature -issues: - - 108762 diff --git a/docs/changelog/109042.yaml b/docs/changelog/109042.yaml deleted file mode 100644 index 5aa80db991c0d..0000000000000 --- a/docs/changelog/109042.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109042 -summary: Add Create or update query rule API call -area: Application -type: enhancement -issues: [ ] diff --git a/docs/changelog/109043.yaml b/docs/changelog/109043.yaml deleted file mode 100644 index bdfe3addea8e9..0000000000000 --- a/docs/changelog/109043.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109043 -summary: "Apm-data: set concrete values for `metricset.interval`" -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/109044.yaml b/docs/changelog/109044.yaml deleted file mode 100644 index 9e50c377606a0..0000000000000 --- a/docs/changelog/109044.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109044 -summary: Enable fallback synthetic source for `token_count` -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/109047.yaml b/docs/changelog/109047.yaml deleted file mode 100644 index 85a8808353a08..0000000000000 --- a/docs/changelog/109047.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109047 -summary: Prevent concurrent jobs during cleanup -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/109070.yaml b/docs/changelog/109070.yaml deleted file mode 100644 index 8dbc0ec1c6cf2..0000000000000 --- a/docs/changelog/109070.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109070 -summary: "ESQL: Add `ip_prefix` function" -area: ES|QL -type: feature -issues: - - 99064 diff --git a/docs/changelog/109071.yaml b/docs/changelog/109071.yaml deleted file mode 100644 index 275a5433cc1d8..0000000000000 --- a/docs/changelog/109071.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109071 -summary: Better handling of multiple rescorers clauses with LTR -area: "Search" -type: bug -issues: [] diff --git a/docs/changelog/109078.yaml b/docs/changelog/109078.yaml deleted file mode 100644 index f602ee9b131bc..0000000000000 --- a/docs/changelog/109078.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109078 -summary: Expose API Key cache metrics -area: Authentication -type: enhancement -issues: [] diff --git a/docs/changelog/109084.yaml b/docs/changelog/109084.yaml deleted file mode 100644 index 67ff5610c5a66..0000000000000 --- a/docs/changelog/109084.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109084 -summary: Add AVX-512 optimised vector distance functions for int7 on x64 -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/109104.yaml b/docs/changelog/109104.yaml deleted file mode 100644 index 985cf14bc5952..0000000000000 --- a/docs/changelog/109104.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109104 -summary: Offload request to generic threadpool -area: Machine Learning -type: bug -issues: - - 109100 diff --git a/docs/changelog/109123.yaml b/docs/changelog/109123.yaml deleted file mode 100644 index dfd7e52b33e7f..0000000000000 --- a/docs/changelog/109123.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109123 -summary: "[Inference API] Check for related pipelines on delete inference endpoint" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/109126.yaml b/docs/changelog/109126.yaml deleted file mode 100644 index 248eacc76b65c..0000000000000 --- a/docs/changelog/109126.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109126 -summary: Correctly handle duplicate model ids for the `_cat` trained models api and usage statistics -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/109167.yaml b/docs/changelog/109167.yaml deleted file mode 100644 index e366b2302263c..0000000000000 --- a/docs/changelog/109167.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109167 -summary: Fixes cluster state-based role mappings not recovered from disk -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/109174.yaml b/docs/changelog/109174.yaml deleted file mode 100644 index 5cd57ebd34ac6..0000000000000 --- a/docs/changelog/109174.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109174 -summary: "ESQL: Change \"substring\" function to not return null on empty string" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/109185.yaml b/docs/changelog/109185.yaml deleted file mode 100644 index 4da72c4b20ffb..0000000000000 --- a/docs/changelog/109185.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109185 -summary: Handle unmatching remote cluster wildcards properly for `IndicesRequest.SingleIndexNoWildcards` - requests -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/109194.yaml b/docs/changelog/109194.yaml deleted file mode 100644 index bf50139547f62..0000000000000 --- a/docs/changelog/109194.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109194 -summary: "[Inference API] Add Mistral Embeddings Support to Inference API" -area: Machine Learning -type: enhancement -issues: [ ] diff --git a/docs/changelog/109196.yaml b/docs/changelog/109196.yaml deleted file mode 100644 index 7f5ca3efbc8d4..0000000000000 --- a/docs/changelog/109196.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109196 -summary: Handle nullable `DocsStats` and `StoresStats` -area: Distributed -type: bug -issues: [] diff --git a/docs/changelog/109204.yaml b/docs/changelog/109204.yaml deleted file mode 100644 index b5b22ef1a06f9..0000000000000 --- a/docs/changelog/109204.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109204 -summary: Detect long-running tasks on network threads -area: Network -type: enhancement -issues: [] diff --git a/docs/changelog/109205.yaml b/docs/changelog/109205.yaml deleted file mode 100644 index 10f13a6549fbc..0000000000000 --- a/docs/changelog/109205.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109205 -summary: "ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs`" -area: ES|QL -type: bug -issues: - - 109198 diff --git a/docs/changelog/109219.yaml b/docs/changelog/109219.yaml deleted file mode 100644 index abf4f49235166..0000000000000 --- a/docs/changelog/109219.yaml +++ /dev/null @@ -1,15 +0,0 @@ -pr: 109219 -summary: Update Lucene version to 9.11 -area: Search -type: feature -issues: [] -highlight: - title: "Update Elasticsearch to Lucene 9.11" - body: |- - Elasticsearch is now updated using the latest Lucene version 9.11. - Here are the full release notes: - But, here are some particular highlights: - - Usage of MADVISE for better memory management: https://github.com/apache/lucene/pull/13196 - - Use RWLock to access LRUQueryCache to reduce contention: https://github.com/apache/lucene/pull/13306 - - Speedup multi-segment HNSW graph search for nested kNN queries: https://github.com/apache/lucene/pull/13121 - - Add a MemorySegment Vector scorer - for scoring without copying on-heap vectors: https://github.com/apache/lucene/pull/13339 diff --git a/docs/changelog/109220.yaml b/docs/changelog/109220.yaml deleted file mode 100644 index b8efa8f784d7a..0000000000000 --- a/docs/changelog/109220.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109220 -summary: "ESQL: add REPEAT string function" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/109233.yaml b/docs/changelog/109233.yaml deleted file mode 100644 index 36010273c80db..0000000000000 --- a/docs/changelog/109233.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109233 -summary: Fix trappy timeouts in security settings APIs -area: Security -type: bug -issues: [] diff --git a/docs/changelog/109236.yaml b/docs/changelog/109236.yaml deleted file mode 100644 index e2eb917ea0343..0000000000000 --- a/docs/changelog/109236.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109236 -summary: Use proper executor for failing requests when connection closes -area: Network -type: bug -issues: - - 109225 diff --git a/docs/changelog/109240.yaml b/docs/changelog/109240.yaml deleted file mode 100644 index a9fad3abdc47f..0000000000000 --- a/docs/changelog/109240.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109240 -summary: Fix trappy timeout in allocation explain API -area: Allocation -type: bug -issues: [] diff --git a/docs/changelog/109241.yaml b/docs/changelog/109241.yaml deleted file mode 100644 index b7343b9df1841..0000000000000 --- a/docs/changelog/109241.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109241 -summary: Fix misc trappy allocation API timeouts -area: Allocation -type: bug -issues: [] diff --git a/docs/changelog/109256.yaml b/docs/changelog/109256.yaml deleted file mode 100644 index 30c15ed77f9b9..0000000000000 --- a/docs/changelog/109256.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 109256 -summary: "[ESQL] Migrate `SimplifyComparisonArithmetics` optimization" -area: ES|QL -type: bug -issues: - - 108388 - - 108743 diff --git a/docs/changelog/109312.yaml b/docs/changelog/109312.yaml deleted file mode 100644 index 594d3f90e8fd1..0000000000000 --- a/docs/changelog/109312.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109312 -summary: Enable fallback synthetic source for `point` and `shape` -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/109317.yaml b/docs/changelog/109317.yaml deleted file mode 100644 index 1d8595d99c2a6..0000000000000 --- a/docs/changelog/109317.yaml +++ /dev/null @@ -1,13 +0,0 @@ -pr: 109317 -summary: Add new int4 quantization to dense_vector -area: Search -type: feature -issues: [] -highlight: - title: Add new int4 quantization to dense_vector - body: |- - New int4 (half-byte) scalar quantization support via two knew index types: `int4_hnsw` and `int4_flat`. - This gives an 8x reduction from `float32` with some accuracy loss. In addition to less memory required, this - improves query and merge speed significantly when compared to raw vectors. - notable: true - diff --git a/docs/changelog/109332.yaml b/docs/changelog/109332.yaml deleted file mode 100644 index 3d03523fd518b..0000000000000 --- a/docs/changelog/109332.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109332 -summary: "ES|QL: vectorize eval" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/109358.yaml b/docs/changelog/109358.yaml deleted file mode 100644 index af47b4129d874..0000000000000 --- a/docs/changelog/109358.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109358 -summary: Use the multi node routing action for internal inference services -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/109359.yaml b/docs/changelog/109359.yaml deleted file mode 100644 index 37202eb5a28ec..0000000000000 --- a/docs/changelog/109359.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109359 -summary: Adding hamming distance function to painless for `dense_vector` fields -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/109370.yaml b/docs/changelog/109370.yaml deleted file mode 100644 index 32b190d1a1c94..0000000000000 --- a/docs/changelog/109370.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109370 -summary: Enable fallback synthetic source by default -area: Mapping -type: feature -issues: - - 106460 diff --git a/docs/changelog/109384.yaml b/docs/changelog/109384.yaml deleted file mode 100644 index 303da23d57d8e..0000000000000 --- a/docs/changelog/109384.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109384 -summary: Fix serialising inference delete response -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/109386.yaml b/docs/changelog/109386.yaml deleted file mode 100644 index 984ee96dde063..0000000000000 --- a/docs/changelog/109386.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109386 -summary: "ESQL: `top_list` aggregation" -area: ES|QL -type: feature -issues: - - 109213 diff --git a/docs/changelog/109395.yaml b/docs/changelog/109395.yaml deleted file mode 100644 index e5336695afa48..0000000000000 --- a/docs/changelog/109395.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109395 -summary: Correct positioning for unique token filter -area: Analysis -type: bug -issues: [] diff --git a/docs/changelog/109410.yaml b/docs/changelog/109410.yaml deleted file mode 100644 index e8c4dcdab42c6..0000000000000 --- a/docs/changelog/109410.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109410 -summary: Support synthetic source for date fields when `ignore_malformed` is used -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/109444.yaml b/docs/changelog/109444.yaml deleted file mode 100644 index 8c56fe2dd9f02..0000000000000 --- a/docs/changelog/109444.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109444 -summary: "Aggs: Scripted metric allow list" -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/109449.yaml b/docs/changelog/109449.yaml deleted file mode 100644 index 90cb908227f1b..0000000000000 --- a/docs/changelog/109449.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109449 -summary: Reset max page size to settings value -area: Transform -type: bug -issues: - - 109308 diff --git a/docs/changelog/109462.yaml b/docs/changelog/109462.yaml deleted file mode 100644 index a05f4a04e80ae..0000000000000 --- a/docs/changelog/109462.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109462 -summary: Add `wait_for_completion` parameter to delete snapshot request -area: Distributed -type: enhancement -issues: - - 101300 diff --git a/docs/changelog/109470.yaml b/docs/changelog/109470.yaml deleted file mode 100644 index 837c1664b775a..0000000000000 --- a/docs/changelog/109470.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109470 -summary: Enabling profiling for `RankBuilders` and adding tests for RRF -area: Ranking -type: enhancement -issues: [] diff --git a/docs/changelog/109480.yaml b/docs/changelog/109480.yaml deleted file mode 100644 index 3a6f48e9bd840..0000000000000 --- a/docs/changelog/109480.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109480 -summary: "[Connector API] Add claim sync job endpoint" -area: Application -type: feature -issues: [] diff --git a/docs/changelog/109481.yaml b/docs/changelog/109481.yaml deleted file mode 100644 index e8251788a90bd..0000000000000 --- a/docs/changelog/109481.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109481 -summary: Fork freeing search/scroll contexts to GENERIC pool -area: Search -type: bug -issues: [] diff --git a/docs/changelog/109487.yaml b/docs/changelog/109487.yaml deleted file mode 100644 index c69c77203f12d..0000000000000 --- a/docs/changelog/109487.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109487 -summary: Start Trained Model Deployment API request query params now override body params -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/109501.yaml b/docs/changelog/109501.yaml deleted file mode 100644 index 6e81f98816cbf..0000000000000 --- a/docs/changelog/109501.yaml +++ /dev/null @@ -1,14 +0,0 @@ -pr: 109501 -summary: Reflect latest changes in synthetic source documentation -area: Mapping -type: enhancement -issues: [] -highlight: - title: Synthetic `_source` improvements - body: |- - There are multiple improvements to synthetic `_source` functionality: - - * Synthetic `_source` is now supported for all field types including `nested` and `object`. `object` fields are supported with `enabled` set to `false`. - - * Synthetic `_source` can be enabled together with `ignore_malformed` and `ignore_above` parameters for all field types that support them. - notable: false diff --git a/docs/changelog/109506.yaml b/docs/changelog/109506.yaml deleted file mode 100644 index 3a7570ed0b93a..0000000000000 --- a/docs/changelog/109506.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109506 -summary: Support synthetic source for `scaled_float` and `unsigned_long` when `ignore_malformed` - is used -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/109534.yaml b/docs/changelog/109534.yaml deleted file mode 100644 index c6eb520bb70a8..0000000000000 --- a/docs/changelog/109534.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109534 -summary: Propagate accurate deployment timeout -area: Machine Learning -type: bug -issues: - - 109407 diff --git a/docs/changelog/109540.yaml b/docs/changelog/109540.yaml deleted file mode 100644 index 722c60a30fb97..0000000000000 --- a/docs/changelog/109540.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109540 -summary: Add metrics@custom component template to metrics-*-* index template -area: Data streams -type: enhancement -issues: - - 109475 diff --git a/docs/changelog/109551.yaml b/docs/changelog/109551.yaml deleted file mode 100644 index f4949669091d9..0000000000000 --- a/docs/changelog/109551.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109551 -summary: Avoid `InferenceRunner` deadlock -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/109554.yaml b/docs/changelog/109554.yaml deleted file mode 100644 index 4e78a8f3044c7..0000000000000 --- a/docs/changelog/109554.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109554 -summary: "[Query Rules] Add API calls to get or delete individual query rules within\ - \ a ruleset" -area: Relevance -type: enhancement -issues: [] diff --git a/docs/changelog/109563.yaml b/docs/changelog/109563.yaml deleted file mode 100644 index 9099064b6b040..0000000000000 --- a/docs/changelog/109563.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109563 -summary: Add allocation explain output for THROTTLING shards -area: Infra/Core -type: enhancement -issues: [] diff --git a/docs/changelog/109597.yaml b/docs/changelog/109597.yaml deleted file mode 100644 index 9b99df85da6a3..0000000000000 --- a/docs/changelog/109597.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109597 -summary: Opt `scripted_metric` out of parallelization -area: Aggregations -type: feature -issues: [] diff --git a/docs/changelog/109603.yaml b/docs/changelog/109603.yaml deleted file mode 100644 index 2d6e8b94aa8d0..0000000000000 --- a/docs/changelog/109603.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109603 -summary: Update translog `writeLocation` for `flushListener` after commit -area: Engine -type: enhancement -issues: [] diff --git a/docs/changelog/109606.yaml b/docs/changelog/109606.yaml deleted file mode 100644 index 6c9089c4c4fde..0000000000000 --- a/docs/changelog/109606.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109606 -summary: Avoid NPE if `users_roles` file does not exist -area: Authentication -type: bug -issues: [] diff --git a/docs/changelog/109613.yaml b/docs/changelog/109613.yaml deleted file mode 100644 index 21d152ac1d6de..0000000000000 --- a/docs/changelog/109613.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109613 -summary: Consider `error_trace` supported by all endpoints -area: Infra/REST API -type: bug -issues: - - 109612 diff --git a/docs/changelog/109618.yaml b/docs/changelog/109618.yaml deleted file mode 100644 index f28bb15a53d96..0000000000000 --- a/docs/changelog/109618.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109618 -summary: Fail cluster state API if blocked -area: Cluster Coordination -type: bug -issues: - - 107503 diff --git a/docs/changelog/109634.yaml b/docs/changelog/109634.yaml deleted file mode 100644 index 4c6358578b6de..0000000000000 --- a/docs/changelog/109634.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109634 -summary: "[Query Rules] Require Enterprise License for Query Rules" -area: Relevance -type: enhancement -issues: [] diff --git a/docs/changelog/109651.yaml b/docs/changelog/109651.yaml deleted file mode 100644 index 982e6a5b536cc..0000000000000 --- a/docs/changelog/109651.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109651 -summary: Support synthetic source for `geo_point` when `ignore_malformed` is used -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/109653.yaml b/docs/changelog/109653.yaml deleted file mode 100644 index 665163ec2a91b..0000000000000 --- a/docs/changelog/109653.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109653 -summary: Handle the "JSON memory allocator bytes" field -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/109657.yaml b/docs/changelog/109657.yaml deleted file mode 100644 index 35b315b7568c9..0000000000000 --- a/docs/changelog/109657.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109657 -summary: Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/109672.yaml b/docs/changelog/109672.yaml deleted file mode 100644 index bb6532ab7accf..0000000000000 --- a/docs/changelog/109672.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109672 -summary: Log repo UUID at generation/registration time -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/109717.yaml b/docs/changelog/109717.yaml deleted file mode 100644 index 326657ea4ce21..0000000000000 --- a/docs/changelog/109717.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109717 -summary: Bump jackson version in modules:repository-azure -area: Snapshot/Restore -type: upgrade -issues: [] diff --git a/docs/changelog/109720.yaml b/docs/changelog/109720.yaml deleted file mode 100644 index b029726c84427..0000000000000 --- a/docs/changelog/109720.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109720 -summary: "DocsStats: Add human readable bytesize" -area: Stats -type: enhancement -issues: [] diff --git a/docs/changelog/109746.yaml b/docs/changelog/109746.yaml deleted file mode 100644 index 5360f545333ac..0000000000000 --- a/docs/changelog/109746.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109746 -summary: ES|QL Add primitive float support to the Compute Engine -area: ES|QL -type: enhancement -issues: - - 109178 diff --git a/docs/changelog/109779.yaml b/docs/changelog/109779.yaml deleted file mode 100644 index 4ccd8d475ec8d..0000000000000 --- a/docs/changelog/109779.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109779 -summary: Include component templates in retention validaiton -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/109781.yaml b/docs/changelog/109781.yaml deleted file mode 100644 index df74645b53d84..0000000000000 --- a/docs/changelog/109781.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109781 -summary: ES|QL Add primitive float variants of all aggregators to the compute engine -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/109794.yaml b/docs/changelog/109794.yaml deleted file mode 100644 index d244c69a903ba..0000000000000 --- a/docs/changelog/109794.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109794 -summary: Provide document size reporter with `MapperService` -area: Infra/Metrics -type: bug -issues: [] diff --git a/docs/changelog/109807.yaml b/docs/changelog/109807.yaml deleted file mode 100644 index 5cf8a2c896c4e..0000000000000 --- a/docs/changelog/109807.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109807 -summary: "ESQL: Fix LOOKUP attribute shadowing" -area: ES|QL -type: bug -issues: - - 109392 diff --git a/docs/changelog/109813.yaml b/docs/changelog/109813.yaml deleted file mode 100644 index edcef17e87606..0000000000000 --- a/docs/changelog/109813.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109813 -summary: Add text similarity reranker retriever -area: Ranking -type: feature -issues: [] diff --git a/docs/changelog/109848.yaml b/docs/changelog/109848.yaml deleted file mode 100644 index 858bbe84ef3a4..0000000000000 --- a/docs/changelog/109848.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109848 -summary: Denser in-memory representation of `ShardBlobsToDelete` -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/109873.yaml b/docs/changelog/109873.yaml deleted file mode 100644 index c77197cc22d0a..0000000000000 --- a/docs/changelog/109873.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109873 -summary: "ESQL: add Arrow dataframes output format" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/109876.yaml b/docs/changelog/109876.yaml deleted file mode 100644 index 4a65b4e17c4a3..0000000000000 --- a/docs/changelog/109876.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109876 -summary: Always pick the user `maxPageSize` value -area: Transform -type: bug -issues: - - 109844 diff --git a/docs/changelog/109880.yaml b/docs/changelog/109880.yaml deleted file mode 100644 index 71c7209824a8a..0000000000000 --- a/docs/changelog/109880.yaml +++ /dev/null @@ -1,10 +0,0 @@ -pr: 109880 -summary: Deprecate `text_expansion` and `weighted_tokens` queries -area: Machine Learning -type: deprecation -issues: [ ] -deprecation: - title: Deprecate `text_expansion` and `weighted_tokens` queries - area: REST API - details: The `text_expansion` and `weighted_tokens` queries have been replaced by `sparse_vector`. - impact: Please update your existing `text_expansion` and `weighted_tokens` queries to use `sparse_vector.` diff --git a/docs/changelog/109882.yaml b/docs/changelog/109882.yaml deleted file mode 100644 index 0f0fed01c5a7a..0000000000000 --- a/docs/changelog/109882.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109882 -summary: Support synthetic source together with `ignore_malformed` in histogram fields -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/109893.yaml b/docs/changelog/109893.yaml deleted file mode 100644 index df6d6e51236c8..0000000000000 --- a/docs/changelog/109893.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109893 -summary: Add Anthropic messages integration to Inference API -area: Machine Learning -type: enhancement -issues: [ ] diff --git a/docs/changelog/109908.yaml b/docs/changelog/109908.yaml deleted file mode 100644 index cdf2acf17096c..0000000000000 --- a/docs/changelog/109908.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109908 -summary: "Update checkpoints after post-replication actions, even on failure" -area: CRUD -type: bug -issues: [] diff --git a/docs/changelog/109931.yaml b/docs/changelog/109931.yaml deleted file mode 100644 index 3575cfd49176f..0000000000000 --- a/docs/changelog/109931.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109931 -summary: Apply FLS to the contents of `IgnoredSourceFieldMapper` -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/109957.yaml b/docs/changelog/109957.yaml deleted file mode 100644 index 6bbcd8175501c..0000000000000 --- a/docs/changelog/109957.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109957 -summary: Add request metric to `RestController` to track success/failure (by status - code) -area: Infra/Metrics -type: enhancement -issues: [] diff --git a/docs/changelog/109963.yaml b/docs/changelog/109963.yaml deleted file mode 100644 index 1745d549582d4..0000000000000 --- a/docs/changelog/109963.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109963 -summary: Propagate mapper builder context flags across nested mapper builder context - creation -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/109967.yaml b/docs/changelog/109967.yaml deleted file mode 100644 index cfc6b6462954b..0000000000000 --- a/docs/changelog/109967.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109967 -summary: Default the HF service to cosine similarity -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/109981.yaml b/docs/changelog/109981.yaml deleted file mode 100644 index cf9388f79e29c..0000000000000 --- a/docs/changelog/109981.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109981 -summary: Limit number of synonym rules that can be created -area: Mapping -type: bug -issues: [108785] diff --git a/docs/changelog/109989.yaml b/docs/changelog/109989.yaml deleted file mode 100644 index f1f5972b60eb3..0000000000000 --- a/docs/changelog/109989.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109989 -summary: "ESQL: Fix Join references" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/109993.yaml b/docs/changelog/109993.yaml deleted file mode 100644 index 40d161b6b5c24..0000000000000 --- a/docs/changelog/109993.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109993 -summary: "[ES|QL] `weighted_avg`" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/110004.yaml b/docs/changelog/110004.yaml deleted file mode 100644 index f680016527a9c..0000000000000 --- a/docs/changelog/110004.yaml +++ /dev/null @@ -1,11 +0,0 @@ -pr: 110004 -summary: Mark Query Rules as GA -area: Relevance -type: feature -issues: [] -highlight: - title: Mark Query Rules as GA - body: |- - This PR marks query rules as Generally Available. All APIs are no longer - in tech preview. - notable: true diff --git a/docs/changelog/110016.yaml b/docs/changelog/110016.yaml deleted file mode 100644 index 28ad55aa796c8..0000000000000 --- a/docs/changelog/110016.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110016 -summary: Opt in keyword field into fallback synthetic source if needed -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/110019.yaml b/docs/changelog/110019.yaml deleted file mode 100644 index 632e79008d351..0000000000000 --- a/docs/changelog/110019.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110019 -summary: Improve mechanism for extracting the result of a `PlainActionFuture` -area: Distributed -type: enhancement -issues: - - 108125 diff --git a/docs/changelog/110046.yaml b/docs/changelog/110046.yaml deleted file mode 100644 index 6ebe440e7aced..0000000000000 --- a/docs/changelog/110046.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110046 -summary: "ESQL: make named params objects truly per request" -area: ES|QL -type: bug -issues: - - 110028 diff --git a/docs/changelog/110059.yaml b/docs/changelog/110059.yaml deleted file mode 100644 index ba160c091cdc2..0000000000000 --- a/docs/changelog/110059.yaml +++ /dev/null @@ -1,32 +0,0 @@ -pr: 110059 -summary: Adds new `bit` `element_type` for `dense_vectors` -area: Vector Search -type: feature -issues: [] -highlight: - title: Adds new `bit` `element_type` for `dense_vectors` - body: |- - This adds `bit` vector support by adding `element_type: bit` for - vectors. This new element type works for indexed and non-indexed - vectors. Additionally, it works with `hnsw` and `flat` index types. No - quantization based codec works with this element type, this is - consistent with `byte` vectors. - - `bit` vectors accept up to `32768` dimensions in size and expect vectors - that are being indexed to be encoded either as a hexidecimal string or a - `byte[]` array where each element of the `byte` array represents `8` - bits of the vector. - - `bit` vectors support script usage and regular query usage. When - indexed, all comparisons done are `xor` and `popcount` summations (aka, - hamming distance), and the scores are transformed and normalized given - the vector dimensions. - - For scripts, `l1norm` is the same as `hamming` distance and `l2norm` is - `sqrt(l1norm)`. `dotProduct` and `cosineSimilarity` are not supported. - - Note, the dimensions expected by this element_type are always to be - divisible by `8`, and the `byte[]` vectors provided for index must be - have size `dim/8` size, where each byte element represents `8` bits of - the vectors. - notable: true diff --git a/docs/changelog/110061.yaml b/docs/changelog/110061.yaml deleted file mode 100644 index 1880a2a197722..0000000000000 --- a/docs/changelog/110061.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110061 -summary: Avoiding running watch jobs in TickerScheduleTriggerEngine if it is paused -area: Watcher -type: bug -issues: - - 105933 diff --git a/docs/changelog/110066.yaml b/docs/changelog/110066.yaml deleted file mode 100644 index 920c6304b63ae..0000000000000 --- a/docs/changelog/110066.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110066 -summary: Support flattened fields and multi-fields as dimensions in downsampling -area: Downsampling -type: bug -issues: - - 99297 diff --git a/docs/changelog/110096.yaml b/docs/changelog/110096.yaml deleted file mode 100644 index 3d6616c289266..0000000000000 --- a/docs/changelog/110096.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110096 -summary: Fix `ClassCastException` with MV_EXPAND on missing field -area: ES|QL -type: bug -issues: - - 109974 diff --git a/docs/changelog/110102.yaml b/docs/changelog/110102.yaml deleted file mode 100644 index d1b9b53e2dfc5..0000000000000 --- a/docs/changelog/110102.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110102 -summary: Optimize ST_DISTANCE filtering with Lucene circle intersection query -area: ES|QL -type: enhancement -issues: - - 109972 diff --git a/docs/changelog/110112.yaml b/docs/changelog/110112.yaml deleted file mode 100644 index eca5fd9af15ce..0000000000000 --- a/docs/changelog/110112.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110112 -summary: Increase response size limit for batched requests -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/110146.yaml b/docs/changelog/110146.yaml deleted file mode 100644 index 61ba35cec319b..0000000000000 --- a/docs/changelog/110146.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110146 -summary: Fix trailing slash in `ml.get_categories` specification -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/110160.yaml b/docs/changelog/110160.yaml deleted file mode 100644 index 0c38c23c69067..0000000000000 --- a/docs/changelog/110160.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110160 -summary: Opt in number fields into fallback synthetic source when doc values a… -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/110176.yaml b/docs/changelog/110176.yaml deleted file mode 100644 index ae1d7d10d6dc4..0000000000000 --- a/docs/changelog/110176.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110176 -summary: Fix trailing slash in two rollup specifications -area: Rollup -type: bug -issues: [] diff --git a/docs/changelog/110177.yaml b/docs/changelog/110177.yaml deleted file mode 100644 index 0ac5328d88df4..0000000000000 --- a/docs/changelog/110177.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110177 -summary: Fix trailing slash in `security.put_privileges` specification -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/110179.yaml b/docs/changelog/110179.yaml deleted file mode 100644 index b99a390c8586f..0000000000000 --- a/docs/changelog/110179.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110179 -summary: Make repository analysis API available to non-operators -area: Snapshot/Restore -type: enhancement -issues: - - 100318 diff --git a/docs/changelog/110186.yaml b/docs/changelog/110186.yaml deleted file mode 100644 index 23eaab118e2ab..0000000000000 --- a/docs/changelog/110186.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110186 -summary: Don't sample calls to `ReduceContext#consumeBucketsAndMaybeBreak` ins `InternalDateHistogram` - and `InternalHistogram` during reduction -area: Aggregations -type: bug -issues: [] diff --git a/docs/changelog/110201.yaml b/docs/changelog/110201.yaml deleted file mode 100644 index a880638881948..0000000000000 --- a/docs/changelog/110201.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110201 -summary: "ES|QL: Fix DISSECT that overwrites input" -area: ES|QL -type: bug -issues: - - 110184 diff --git a/docs/changelog/110214.yaml b/docs/changelog/110214.yaml deleted file mode 100644 index 20f61cac64454..0000000000000 --- a/docs/changelog/110214.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110214 -summary: Handle `ignore_above` in synthetic source for flattened fields -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/110233.yaml b/docs/changelog/110233.yaml deleted file mode 100644 index d9ce4057090a4..0000000000000 --- a/docs/changelog/110233.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110233 -summary: Support k parameter for knn query -area: Vector Search -type: enhancement -issues: - - 108473 diff --git a/docs/changelog/110234.yaml b/docs/changelog/110234.yaml deleted file mode 100644 index 0656ba5fb6636..0000000000000 --- a/docs/changelog/110234.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110234 -summary: Upgrade to Lucene-9.11.1 -area: Search -type: upgrade -issues: [] diff --git a/docs/changelog/110236.yaml b/docs/changelog/110236.yaml deleted file mode 100644 index e2dbff7fbf768..0000000000000 --- a/docs/changelog/110236.yaml +++ /dev/null @@ -1,21 +0,0 @@ -pr: 110236 -summary: '`ParseHeapRatioOrDeprecatedByteSizeValue` for `indices.breaker.total.limit`' -area: Infra/Settings -type: deprecation -issues: [] -deprecation: - title: 'Deprecate absolute size values for `indices.breaker.total.limit` setting' - area: Cluster and node setting - details: Previously, the value of `indices.breaker.total.limit` could be specified as - an absolute size in bytes. This setting controls the overal amount of - memory the server is allowed to use before taking remedial actions. Setting - this to a specific number of bytes led to strange behaviour when the node - maximum heap size changed because the circut breaker limit would remain - unchanged. This would either leave the value too low, causing part of the - heap to remain unused; or it would leave the value too high, causing the - circuit breaker to be ineffective at preventing OOM errors. The only - reasonable behaviour for this setting is that it scales with the size of - the heap, and so absolute byte limits are now deprecated. - impact: Users must change their configuration to specify a percentage instead of - an absolute number of bytes for `indices.breaker.total.limit`, or else - accept the default, which is already specified as a percentage. diff --git a/docs/changelog/110248.yaml b/docs/changelog/110248.yaml deleted file mode 100644 index 85739528b69c6..0000000000000 --- a/docs/changelog/110248.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110248 -summary: "[Inference API] Add Amazon Bedrock Support to Inference API" -area: Machine Learning -type: enhancement -issues: [ ] diff --git a/docs/changelog/110251.yaml b/docs/changelog/110251.yaml deleted file mode 100644 index a3b0c3128be35..0000000000000 --- a/docs/changelog/110251.yaml +++ /dev/null @@ -1,13 +0,0 @@ -pr: 110251 -summary: Support index sorting with nested fields -area: Logs -type: enhancement -issues: - - 107349 -highlight: - title: Index sorting on indexes with nested fields - body: |- - Index sorting is now supported for indexes with mappings containing nested objects. - The index sort spec (as specified by `index.sort.field`) can't contain any nested - fields, still. - notable: false diff --git a/docs/changelog/110334.yaml b/docs/changelog/110334.yaml deleted file mode 100644 index f83ac04ded773..0000000000000 --- a/docs/changelog/110334.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110334 -summary: Sentence Chunker -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/110337.yaml b/docs/changelog/110337.yaml deleted file mode 100644 index bf21a95c9157f..0000000000000 --- a/docs/changelog/110337.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110337 -summary: Support `ignore_above` on keyword dimensions -area: TSDB -type: enhancement -issues: [] diff --git a/docs/changelog/110338.yaml b/docs/changelog/110338.yaml deleted file mode 100644 index 2334a1cbc9283..0000000000000 --- a/docs/changelog/110338.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110338 -summary: Add `semantic_text` field type and `semantic` query -area: Mapping -type: feature -issues: [] diff --git a/docs/changelog/110347.yaml b/docs/changelog/110347.yaml deleted file mode 100644 index 8727128230935..0000000000000 --- a/docs/changelog/110347.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110347 -summary: "ESQL: Renamed `TopList` to Top" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/110361.yaml b/docs/changelog/110361.yaml deleted file mode 100644 index 8558c88e06049..0000000000000 --- a/docs/changelog/110361.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 110361 -summary: Don't detect `PlainActionFuture` deadlock on concurrent complete -area: Distributed -type: bug -issues: - - 110181 - - 110360 diff --git a/docs/changelog/110369.yaml b/docs/changelog/110369.yaml deleted file mode 100644 index 770294605b444..0000000000000 --- a/docs/changelog/110369.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110369 -summary: Run terms concurrently when cardinality is only lower than shard size -area: Aggregations -type: bug -issues: - - 105505 diff --git a/docs/changelog/110383.yaml b/docs/changelog/110383.yaml deleted file mode 100644 index 5e9bddd4bfcd2..0000000000000 --- a/docs/changelog/110383.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110383 -summary: Add bulk delete roles API -area: Security -type: enhancement -issues: [] diff --git a/docs/changelog/110391.yaml b/docs/changelog/110391.yaml deleted file mode 100644 index 1e00eda970398..0000000000000 --- a/docs/changelog/110391.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110391 -summary: Fix ST_DISTANCE Lucene push-down for complex predicates -area: ES|QL -type: bug -issues: - - 110349 diff --git a/docs/changelog/110395.yaml b/docs/changelog/110395.yaml deleted file mode 100644 index 690be55abb5b2..0000000000000 --- a/docs/changelog/110395.yaml +++ /dev/null @@ -1,9 +0,0 @@ -pr: 110395 -summary: Mark the Redact processor as Generally Available -area: Ingest Node -type: feature -issues: [] -highlight: - title: The Redact processor is Generally Available - body: The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns. The Redact processor was initially released as Technical Preview in `8.7.0`, and is now released as Generally Available. - notable: true diff --git a/docs/changelog/110431.yaml b/docs/changelog/110431.yaml deleted file mode 100644 index 0dd93ef718ef9..0000000000000 --- a/docs/changelog/110431.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110431 -summary: "[Inference API] Fix serialization for inference delete endpoint response" -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/110476.yaml b/docs/changelog/110476.yaml deleted file mode 100644 index bc12b3711a366..0000000000000 --- a/docs/changelog/110476.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 110476 -summary: Fix bug in union-types with type-casting in grouping key of STATS -area: ES|QL -type: bug -issues: - - 109922 - - 110477 diff --git a/docs/changelog/110488.yaml b/docs/changelog/110488.yaml deleted file mode 100644 index fbb439f20fc96..0000000000000 --- a/docs/changelog/110488.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110488 -summary: "ESQL: Validate unique plan attribute names" -area: ES|QL -type: bug -issues: - - 110541 diff --git a/docs/changelog/110540.yaml b/docs/changelog/110540.yaml deleted file mode 100644 index 5e4994da80704..0000000000000 --- a/docs/changelog/110540.yaml +++ /dev/null @@ -1,16 +0,0 @@ -pr: 110540 -summary: Deprecate using slm privileges to access ilm -area: ILM+SLM -type: deprecation -issues: [] -deprecation: - title: Deprecate using slm privileges to access ilm - area: REST API - details: The `read_slm` privilege can get the ILM status, and - the `manage_slm` privilege can start and stop ILM. Access to these - APIs should be granted using the `read_ilm` and `manage_ilm` privileges - instead. Access to ILM APIs will be removed from SLM privileges in - a future major release, and is now deprecated. - impact: Users that need access to the ILM status API should now - use the `read_ilm` privilege. Users that need to start and stop ILM, - should use the `manage_ilm` privilege. diff --git a/docs/changelog/110586.yaml b/docs/changelog/110586.yaml deleted file mode 100644 index cc2bcb85a2dac..0000000000000 --- a/docs/changelog/110586.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110586 -summary: "ESQL: Fix Max doubles bug with negatives and add tests for Max and Min" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/110651.yaml b/docs/changelog/110651.yaml deleted file mode 100644 index c25c63ee0284a..0000000000000 --- a/docs/changelog/110651.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110651 -summary: "Remove `default_field: message` from metrics index templates" -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/110665.yaml b/docs/changelog/110665.yaml deleted file mode 100644 index fa6db3190fe60..0000000000000 --- a/docs/changelog/110665.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110665 -summary: "[ESQL] Fix parsing of large magnitude negative numbers" -area: ES|QL -type: bug -issues: - - 104323 diff --git a/docs/changelog/110666.yaml b/docs/changelog/110666.yaml deleted file mode 100644 index d96f8e2024c81..0000000000000 --- a/docs/changelog/110666.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110666 -summary: Removing the use of Stream::peek from `GeoIpDownloader::cleanDatabases` -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/110707.yaml b/docs/changelog/110707.yaml deleted file mode 100644 index e13688c73c743..0000000000000 --- a/docs/changelog/110707.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110707 -summary: Fix issue with returning incomplete fragment for plain highlighter -area: Highlighting -type: bug -issues: [] diff --git a/docs/changelog/110710.yaml b/docs/changelog/110710.yaml deleted file mode 100644 index bf3349ee25cdd..0000000000000 --- a/docs/changelog/110710.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110710 -summary: Add a cluster listener to fix missing node features after upgrading from a version prior to 8.13 -area: Infra/Core -type: bug -issues: - - 109254 diff --git a/docs/changelog/110793.yaml b/docs/changelog/110793.yaml deleted file mode 100644 index 8f1f3ba9afeb7..0000000000000 --- a/docs/changelog/110793.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 110793 -summary: Fix for union-types for multiple columns with the same name -area: ES|QL -type: bug -issues: - - 110490 - - 109916 diff --git a/docs/changelog/110824.yaml b/docs/changelog/110824.yaml deleted file mode 100644 index 4fe97d6692865..0000000000000 --- a/docs/changelog/110824.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110824 -summary: "[ESQL] Count_distinct(_source) should return a 400" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/110844.yaml b/docs/changelog/110844.yaml deleted file mode 100644 index ea879f13f3e67..0000000000000 --- a/docs/changelog/110844.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110844 -summary: Directly download commercial ip geolocation databases from providers -area: Ingest Node -type: feature -issues: [] diff --git a/docs/changelog/110906.yaml b/docs/changelog/110906.yaml deleted file mode 100644 index 6123b1108fd17..0000000000000 --- a/docs/changelog/110906.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110906 -summary: "Add comma before charset parameter in WWW-Authenticate response header" -area: Authentication -type: bug -issues: [] diff --git a/docs/changelog/110922.yaml b/docs/changelog/110922.yaml deleted file mode 100644 index 6a85ce57de103..0000000000000 --- a/docs/changelog/110922.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110922 -summary: Speed up collecting zero document string terms -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/110927.yaml b/docs/changelog/110927.yaml deleted file mode 100644 index 3602ce3e811fa..0000000000000 --- a/docs/changelog/110927.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110927 -summary: Fix leak in collapsing search results -area: Search -type: bug -issues: [] diff --git a/docs/changelog/111184.yaml b/docs/changelog/111184.yaml deleted file mode 100644 index 5ecdba54b09be..0000000000000 --- a/docs/changelog/111184.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111184 -summary: Fix Dissect with leading non-ascii characters -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/111186.yaml b/docs/changelog/111186.yaml deleted file mode 100644 index 3676beb3910c5..0000000000000 --- a/docs/changelog/111186.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111186 -summary: "ES|QL: reduce max expression depth to 400" -area: ES|QL -type: bug -issues: - - 109846 diff --git a/docs/changelog/111290.yaml b/docs/changelog/111290.yaml deleted file mode 100644 index efcb01a4aedf9..0000000000000 --- a/docs/changelog/111290.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111290 -summary: Fix enrich policy runner exception handling on empty segments response -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/111363.yaml b/docs/changelog/111363.yaml deleted file mode 100644 index 2cb3c5342ea5c..0000000000000 --- a/docs/changelog/111363.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111363 -summary: Ensure vector similarity correctly limits `inner_hits` returned for nested - kNN -area: Vector Search -type: bug -issues: [111093] diff --git a/docs/changelog/111366.yaml b/docs/changelog/111366.yaml deleted file mode 100644 index 9cb127077094f..0000000000000 --- a/docs/changelog/111366.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111366 -summary: "[Inference API] Replace `model_id` with `inference_id` in inference API\ - \ except when stored" -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/111369.yaml b/docs/changelog/111369.yaml deleted file mode 100644 index 1a638abea4e1d..0000000000000 --- a/docs/changelog/111369.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111369 -summary: Improve security-crypto threadpool overflow handling -area: Authentication -type: bug -issues: [] From e6cb43b39b3e8fede3d9bd8bed3ff62dd1de9172 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 9 Aug 2024 13:09:05 -0400 Subject: [PATCH 27/37] Add known issue for #111679 (#111760) and restore the pytorch known issue --- docs/reference/release-notes/8.15.0.asciidoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index 40d5cf02954bf..1df0969ecc629 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -3,6 +3,19 @@ Also see <>. + +[[known-issues-8.15.0]] +[float] +=== Known issues +* The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory. +In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference` +process to reclaim memory. This can cause inference requests to fail. +Elasticsearch will automatically restart the `pytorch_inference` process +after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530]) + +* Pipeline aggregations under `time_series` and `categorize_text` aggregations are never +returned (issue: {es-issue}111679[#111679]) + [[breaking-8.15.0]] [float] === Breaking changes From 6b880658b2db581021cb29b457b981928c672b84 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 9 Aug 2024 13:22:52 -0400 Subject: [PATCH 28/37] Revert "Avoid bucket copies in Aggs (#110261)" (#111758) This reverts #110261 which we can't land until #111757 - we need to be sure that the `equals` implementations on subclasses of `InternalAggregations` is correct before this optimization is safe. Closes #111679 --- docs/changelog/111758.yaml | 6 ++++ .../aggregations/InternalAggregations.java | 31 +++---------------- .../InternalMultiBucketAggregation.java | 29 +++++------------ .../AbstractHistogramAggregator.java | 3 -- .../histogram/DateHistogramAggregator.java | 3 -- .../bucket/histogram/InternalHistogram.java | 18 +++++------ 6 files changed, 24 insertions(+), 66 deletions(-) create mode 100644 docs/changelog/111758.yaml diff --git a/docs/changelog/111758.yaml b/docs/changelog/111758.yaml new file mode 100644 index 0000000000000..c95cdf48bc8a7 --- /dev/null +++ b/docs/changelog/111758.yaml @@ -0,0 +1,6 @@ +pr: 111758 +summary: Revert "Avoid bucket copies in Aggs" +area: Aggregations +type: bug +issues: + - 111679 diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java index 297bb81b27b25..4f234c33b13a6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java @@ -226,34 +226,11 @@ public static InternalAggregations topLevelReduce(List agg } if (context.isFinalReduce()) { List reducedInternalAggs = reduced.getInternalAggregations(); - List internalAggregations = null; - for (int i = 0; i < reducedInternalAggs.size(); i++) { - InternalAggregation agg = reducedInternalAggs.get(i); - InternalAggregation internalAggregation = agg.reducePipelines( - agg, - context, - context.pipelineTreeRoot().subTree(agg.getName()) - ); - if (internalAggregation.equals(agg) == false) { - if (internalAggregations == null) { - internalAggregations = new ArrayList<>(reducedInternalAggs); - } - internalAggregations.set(i, internalAggregation); - } - } + reducedInternalAggs = reducedInternalAggs.stream() + .map(agg -> agg.reducePipelines(agg, context, context.pipelineTreeRoot().subTree(agg.getName()))) + .collect(Collectors.toCollection(ArrayList::new)); - var pipelineAggregators = context.pipelineTreeRoot().aggregators(); - if (pipelineAggregators.isEmpty()) { - if (internalAggregations == null) { - return reduced; - } - return from(internalAggregations); - } - if (internalAggregations != null) { - reducedInternalAggs = internalAggregations; - } - reducedInternalAggs = new ArrayList<>(reducedInternalAggs); - for (PipelineAggregator pipelineAggregator : pipelineAggregators) { + for (PipelineAggregator pipelineAggregator : context.pipelineTreeRoot().aggregators()) { SiblingPipelineAggregator sib = (SiblingPipelineAggregator) pipelineAggregator; InternalAggregation newAgg = sib.doReduce(from(reducedInternalAggs), context); reducedInternalAggs.add(newAgg); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java index e046b5fc9244c..de19c26daff92 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java @@ -207,31 +207,16 @@ public void forEachBucket(Consumer consumer) { } private List reducePipelineBuckets(AggregationReduceContext reduceContext, PipelineTree pipelineTree) { - List reducedBuckets = null; - var buckets = getBuckets(); - for (int bucketIndex = 0; bucketIndex < buckets.size(); bucketIndex++) { - B bucket = buckets.get(bucketIndex); - List aggs = null; - int aggIndex = 0; - for (InternalAggregation agg : bucket.getAggregations()) { + List reducedBuckets = new ArrayList<>(); + for (B bucket : getBuckets()) { + List aggs = new ArrayList<>(); + for (Aggregation agg : bucket.getAggregations()) { PipelineTree subTree = pipelineTree.subTree(agg.getName()); - var reduced = agg.reducePipelines(agg, reduceContext, subTree); - if (reduced.equals(agg) == false) { - if (aggs == null) { - aggs = bucket.getAggregations().copyResults(); - } - aggs.set(aggIndex, reduced); - } - aggIndex++; - } - if (aggs != null) { - if (reducedBuckets == null) { - reducedBuckets = new ArrayList<>(buckets); - } - reducedBuckets.set(bucketIndex, createBucket(InternalAggregations.from(aggs), bucket)); + aggs.add(((InternalAggregation) agg).reducePipelines((InternalAggregation) agg, reduceContext, subTree)); } + reducedBuckets.add(createBucket(InternalAggregations.from(aggs), bucket)); } - return reducedBuckets == null ? buckets : reducedBuckets; + return reducedBuckets; } public abstract static class InternalBucket implements Bucket, Writeable { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java index 04028de5656ca..62b7a0747ca00 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java @@ -84,9 +84,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I double key = roundKey * interval + offset; return new InternalHistogram.Bucket(key, docCount, keyed, formatter, subAggregationResults); }, (owningBucketOrd, buckets) -> { - if (buckets.isEmpty()) { - return buildEmptyAggregation(); - } // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java index cb01aa5a31a9a..2c57bd4b38a04 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java @@ -340,9 +340,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I return buildAggregationsForVariableBuckets(owningBucketOrds, bucketOrds, (bucketValue, docCount, subAggregationResults) -> { return new InternalDateHistogram.Bucket(bucketValue, docCount, keyed, formatter, subAggregationResults); }, (owningBucketOrd, buckets) -> { - if (buckets.isEmpty()) { - return buildEmptyAggregation(); - } // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java index b09c84a80ac2c..8d7a7f4b760cc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java @@ -259,17 +259,11 @@ BucketOrder getOrder() { @Override public InternalHistogram create(List buckets) { - if (this.buckets.equals(buckets)) { - return this; - } return new InternalHistogram(name, buckets, order, minDocCount, emptyBucketInfo, format, keyed, metadata); } @Override public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) { - if (prototype.aggregations.equals(aggregations)) { - return prototype; - } return new Bucket(prototype.key, prototype.docCount, prototype.keyed, prototype.format, aggregations); } @@ -456,9 +450,6 @@ public InternalAggregation get() { CollectionUtil.introSort(reducedBuckets, order.comparator()); } } - if (reducedBuckets.equals(buckets)) { - return InternalHistogram.this; - } return new InternalHistogram(getName(), reducedBuckets, order, minDocCount, emptyBucketInfo, format, keyed, getMetadata()); } }; @@ -504,9 +495,14 @@ public Number getKey(MultiBucketsAggregation.Bucket bucket) { } @Override - @SuppressWarnings({ "rawtypes", "unchecked" }) public InternalAggregation createAggregation(List buckets) { - return new InternalHistogram(name, (List) buckets, order, minDocCount, emptyBucketInfo, format, keyed, getMetadata()); + // convert buckets to the right type + List buckets2 = new ArrayList<>(buckets.size()); + for (Object b : buckets) { + buckets2.add((Bucket) b); + } + buckets2 = Collections.unmodifiableList(buckets2); + return new InternalHistogram(name, buckets2, order, minDocCount, emptyBucketInfo, format, keyed, getMetadata()); } @Override From 4e26114764df6a6e93e62701e5f205bcc9e0dad4 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Fri, 9 Aug 2024 13:59:57 -0400 Subject: [PATCH 29/37] Fix NullPointerException when doing knn search on empty index without dims (#111756) * Fix NullPointerException when doing knn search on empty index without dims * Update docs/changelog/111756.yaml * Fix typo in yaml test --------- Co-authored-by: Elastic Machine --- docs/changelog/111756.yaml | 6 + .../test/search.vectors/40_knn_search.yml | 214 ++++++++++-------- .../vectors/DenseVectorFieldMapper.java | 8 +- 3 files changed, 131 insertions(+), 97 deletions(-) create mode 100644 docs/changelog/111756.yaml diff --git a/docs/changelog/111756.yaml b/docs/changelog/111756.yaml new file mode 100644 index 0000000000000..e58345dbe696a --- /dev/null +++ b/docs/changelog/111756.yaml @@ -0,0 +1,6 @@ +pr: 111756 +summary: Fix `NullPointerException` when doing knn search on empty index without dims +area: Vector Search +type: bug +issues: + - 111733 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml index df5d451e3a2e1..8c0e1f45cf305 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml @@ -29,14 +29,24 @@ setup: m: 16 ef_construction: 200 + - do: + indices.create: + index: test_empty + body: + mappings: + properties: + vector: + type: dense_vector + + - do: index: index: test id: "1" body: name: cow.jpg - vector: [230.0, 300.33, -34.8988, 15.555, -200.0] - another_vector: [130.0, 115.0, -1.02, 15.555, -100.0] + vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] + another_vector: [ 130.0, 115.0, -1.02, 15.555, -100.0 ] - do: index: @@ -44,8 +54,8 @@ setup: id: "2" body: name: moose.jpg - vector: [-0.5, 100.0, -13, 14.8, -156.0] - another_vector: [-0.5, 50.0, -1, 1, 120] + vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] + another_vector: [ -0.5, 50.0, -1, 1, 120 ] - do: index: @@ -53,11 +63,11 @@ setup: id: "3" body: name: rabbit.jpg - vector: [0.5, 111.3, -13.0, 14.8, -156.0] - another_vector: [-0.5, 11.0, 0, 12, 111.0] + vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] + another_vector: [ -0.5, 11.0, 0, 12, 111.0 ] - do: - indices.refresh: {} + indices.refresh: { } --- "kNN search only": @@ -71,15 +81,15 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 - - match: {hits.hits.0._id: "2"} - - match: {hits.hits.0.fields.name.0: "moose.jpg"} + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } - - match: {hits.hits.1._id: "3"} - - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.fields.name.0: "rabbit.jpg" } --- "kNN multi-field search only": - requires: @@ -91,14 +101,14 @@ setup: body: fields: [ "name" ] knn: - - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} - - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + - { field: vector, query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ], k: 2, num_candidates: 3 } + - { field: another_vector, query_vector: [ -0.5, 11.0, 0, 12, 111.0 ], k: 2, num_candidates: 3 } - - match: {hits.hits.0._id: "3"} - - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } - - match: {hits.hits.1._id: "2"} - - match: {hits.hits.1.fields.name.0: "moose.jpg"} + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.fields.name.0: "moose.jpg" } --- "kNN search plus query": - requires: @@ -111,21 +121,21 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 query: term: name: cow.jpg - - match: {hits.hits.0._id: "1"} - - match: {hits.hits.0.fields.name.0: "cow.jpg"} + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.fields.name.0: "cow.jpg" } - - match: {hits.hits.1._id: "2"} - - match: {hits.hits.1.fields.name.0: "moose.jpg"} + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.fields.name.0: "moose.jpg" } - - match: {hits.hits.2._id: "3"} - - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} + - match: { hits.hits.2._id: "3" } + - match: { hits.hits.2.fields.name.0: "rabbit.jpg" } --- "kNN multi-field search with query": - requires: @@ -137,20 +147,20 @@ setup: body: fields: [ "name" ] knn: - - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} - - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + - { field: vector, query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ], k: 2, num_candidates: 3 } + - { field: another_vector, query_vector: [ -0.5, 11.0, 0, 12, 111.0 ], k: 2, num_candidates: 3 } query: term: name: cow.jpg - - match: {hits.hits.0._id: "3"} - - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } - - match: {hits.hits.1._id: "1"} - - match: {hits.hits.1.fields.name.0: "cow.jpg"} + - match: { hits.hits.1._id: "1" } + - match: { hits.hits.1.fields.name.0: "cow.jpg" } - - match: {hits.hits.2._id: "2"} - - match: {hits.hits.2.fields.name.0: "moose.jpg"} + - match: { hits.hits.2._id: "2" } + - match: { hits.hits.2.fields.name.0: "moose.jpg" } --- "kNN search with filter": - requires: @@ -163,16 +173,16 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 filter: term: name: "rabbit.jpg" - - match: {hits.total.value: 1} - - match: {hits.hits.0._id: "3"} - - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } - do: search: @@ -181,7 +191,7 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 filter: @@ -190,7 +200,7 @@ setup: - term: _id: 2 - - match: {hits.total.value: 0} + - match: { hits.total.value: 0 } --- "kNN search with explicit search_type": @@ -206,7 +216,7 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 @@ -216,7 +226,7 @@ setup: --- "kNN search in _knn_search endpoint": - skip: - features: ["allowed_warnings"] + features: [ "allowed_warnings" ] - do: allowed_warnings: - "The kNN search API has been replaced by the `knn` option in the search API." @@ -226,22 +236,22 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 - - match: {hits.hits.0._id: "2"} - - match: {hits.hits.0.fields.name.0: "moose.jpg"} + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } - - match: {hits.hits.1._id: "3"} - - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.fields.name.0: "rabbit.jpg" } --- "kNN search with filter in _knn_search endpoint": - requires: cluster_features: "gte_v8.2.0" reason: 'kNN with filtering added in 8.2' - test_runner_features: ["allowed_warnings"] + test_runner_features: [ "allowed_warnings" ] - do: allowed_warnings: - "The kNN search API has been replaced by the `knn` option in the search API." @@ -251,16 +261,16 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 filter: term: name: "rabbit.jpg" - - match: {hits.total.value: 1} - - match: {hits.hits.0._id: "3"} - - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } - do: allowed_warnings: @@ -271,7 +281,7 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 2 num_candidates: 3 filter: @@ -280,7 +290,7 @@ setup: - term: _id: 2 - - match: {hits.total.value: 0} + - match: { hits.total.value: 0 } --- "Test nonexistent field is match none": @@ -298,7 +308,7 @@ setup: k: 2 num_candidates: 3 - - length: {hits.hits: 0} + - length: { hits.hits: 0 } - do: indices.create: @@ -347,12 +357,12 @@ setup: k: 3 field: vector similarity: 11 - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] - - length: {hits.hits: 1} + - length: { hits.hits: 1 } - - match: {hits.hits.0._id: "2"} - - match: {hits.hits.0.fields.name.0: "moose.jpg"} + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } --- "Vector similarity with filter only": - requires: @@ -368,13 +378,13 @@ setup: k: 3 field: vector similarity: 11 - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] - filter: {"term": {"name": "moose.jpg"}} + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + filter: { "term": { "name": "moose.jpg" } } - - length: {hits.hits: 1} + - length: { hits.hits: 1 } - - match: {hits.hits.0._id: "2"} - - match: {hits.hits.0.fields.name.0: "moose.jpg"} + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } - do: search: @@ -386,10 +396,10 @@ setup: k: 3 field: vector similarity: 110 - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] - filter: {"term": {"name": "cow.jpg"}} + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + filter: { "term": { "name": "cow.jpg" } } - - length: {hits.hits: 0} + - length: { hits.hits: 0 } --- "Knn search with mip": - requires: @@ -421,7 +431,7 @@ setup: id: "1" body: name: cow.jpg - vector: [230.0, 300.33, -34.8988, 15.555, -200.0] + vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] - do: index: @@ -429,7 +439,7 @@ setup: id: "2" body: name: moose.jpg - vector: [-0.5, 100.0, -13, 14.8, -156.0] + vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] - do: index: @@ -437,10 +447,10 @@ setup: id: "3" body: name: rabbit.jpg - vector: [0.5, 111.3, -13.0, 14.8, -156.0] + vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] - do: - indices.refresh: {} + indices.refresh: { } - do: search: @@ -451,16 +461,16 @@ setup: num_candidates: 3 k: 3 field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] - - length: {hits.hits: 3} - - match: {hits.hits.0._id: "1"} - - close_to: {hits.hits.0._score: {value: 58694.902, error: 0.01}} - - match: {hits.hits.1._id: "3"} - - close_to: {hits.hits.1._score: {value: 34702.79, error: 0.01}} - - match: {hits.hits.2._id: "2"} - - close_to: {hits.hits.2._score: {value: 33686.29, error: 0.01}} + - length: { hits.hits: 3 } + - match: { hits.hits.0._id: "1" } + - close_to: { hits.hits.0._score: { value: 58694.902, error: 0.01 } } + - match: { hits.hits.1._id: "3" } + - close_to: { hits.hits.1._score: { value: 34702.79, error: 0.01 } } + - match: { hits.hits.2._id: "2" } + - close_to: { hits.hits.2._score: { value: 33686.29, error: 0.01 } } - do: search: @@ -471,14 +481,14 @@ setup: num_candidates: 3 k: 3 field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] filter: { "term": { "name": "moose.jpg" } } - - length: {hits.hits: 1} - - match: {hits.hits.0._id: "2"} - - close_to: {hits.hits.0._score: {value: 33686.29, error: 0.01}} + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "2" } + - close_to: { hits.hits.0._score: { value: 33686.29, error: 0.01 } } --- "Knn search with _name": - requires: @@ -493,7 +503,7 @@ setup: fields: [ "name" ] knn: field: vector - query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] k: 3 num_candidates: 3 _name: "my_knn_query" @@ -504,15 +514,33 @@ setup: _name: "my_query" - - match: {hits.hits.0._id: "1"} - - match: {hits.hits.0.fields.name.0: "cow.jpg"} - - match: {hits.hits.0.matched_queries.0: "my_knn_query"} - - match: {hits.hits.0.matched_queries.1: "my_query"} + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.fields.name.0: "cow.jpg" } + - match: { hits.hits.0.matched_queries.0: "my_knn_query" } + - match: { hits.hits.0.matched_queries.1: "my_query" } - - match: {hits.hits.1._id: "2"} - - match: {hits.hits.1.fields.name.0: "moose.jpg"} - - match: {hits.hits.1.matched_queries.0: "my_knn_query"} + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.fields.name.0: "moose.jpg" } + - match: { hits.hits.1.matched_queries.0: "my_knn_query" } + + - match: { hits.hits.2._id: "3" } + - match: { hits.hits.2.fields.name.0: "rabbit.jpg" } + - match: { hits.hits.2.matched_queries.0: "my_knn_query" } + +--- +"kNN search on empty index should return 0 results and not an error": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'Error fixed in 8.16.0' + - do: + search: + index: test_empty + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 - - match: {hits.hits.2._id: "3"} - - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} - - match: {hits.hits.2.matched_queries.0: "my_knn_query"} + - match: { hits.total.value: 0 } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 81fb7990f09eb..6a231d15f7be8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -937,8 +937,8 @@ int parseDimensionCount(DocumentParserContext context) throws IOException { } @Override - public void checkDimensions(int dvDims, int qvDims) { - if (dvDims != qvDims * Byte.SIZE) { + public void checkDimensions(Integer dvDims, int qvDims) { + if (dvDims != null && dvDims != qvDims * Byte.SIZE) { throw new IllegalArgumentException( "The query vector has a different number of dimensions [" + qvDims * Byte.SIZE @@ -972,8 +972,8 @@ abstract void checkVectorMagnitude( float squaredMagnitude ); - public void checkDimensions(int dvDims, int qvDims) { - if (dvDims != qvDims) { + public void checkDimensions(Integer dvDims, int qvDims) { + if (dvDims != null && dvDims != qvDims) { throw new IllegalArgumentException( "The query vector has a different number of dimensions [" + qvDims + "] than the document vectors [" + dvDims + "]." ); From 98b18798b0fc8dc3e94cd396c2408e10829e5ef0 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 9 Aug 2024 14:05:46 -0400 Subject: [PATCH 30/37] ESQL: Javadocs for DataTypes (#111649) This adds some javadocs for a few of the `DataTypes` so if you mouseover them you'll get a bit of a description. It also explains a bit more about how we load `TEXT` fields which are pretty unique. --- .../xpack/esql/core/type/DataType.java | 61 +++++++++++++++++-- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 7bf9ea9b28767..9b1c0e710a9d7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -36,18 +36,42 @@ public enum DataType { * rendered as {@code null} in the response. */ UNSUPPORTED(builder().typeName("UNSUPPORTED").unknownSize()), + /** + * Fields that are always {@code null}, usually created with constant + * {@code null} values. + */ NULL(builder().esType("null").estimatedSize(0)), + /** + * Fields that can either be {@code true} or {@code false}. + */ BOOLEAN(builder().esType("boolean").estimatedSize(1)), /** - * These are numeric fields labeled as metric counters in time-series indices. Although stored - * internally as numeric fields, they represent cumulative metrics and must not be treated as regular - * numeric fields. Therefore, we define them differently and separately from their parent numeric field. - * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their - * parent numeric type. + * 64-bit signed numbers labeled as metric counters in time-series indices. + * Although stored internally as numeric fields, they represent cumulative + * metrics and must not be treated as regular numeric fields. Therefore, + * we define them differently and separately from their parent numeric field. + * These fields are strictly for use in retrieval from indices, rate + * aggregation, and casting to their parent numeric type. */ COUNTER_LONG(builder().esType("counter_long").estimatedSize(Long.BYTES).docValues().counter()), + /** + * 32-bit signed numbers labeled as metric counters in time-series indices. + * Although stored internally as numeric fields, they represent cumulative + * metrics and must not be treated as regular numeric fields. Therefore, + * we define them differently and separately from their parent numeric field. + * These fields are strictly for use in retrieval from indices, rate + * aggregation, and casting to their parent numeric type. + */ COUNTER_INTEGER(builder().esType("counter_integer").estimatedSize(Integer.BYTES).docValues().counter()), + /** + * 64-bit floating point numbers labeled as metric counters in time-series indices. + * Although stored internally as numeric fields, they represent cumulative + * metrics and must not be treated as regular numeric fields. Therefore, + * we define them differently and separately from their parent numeric field. + * These fields are strictly for use in retrieval from indices, rate + * aggregation, and casting to their parent numeric type. + */ COUNTER_DOUBLE(builder().esType("counter_double").estimatedSize(Double.BYTES).docValues().counter()), /** @@ -98,14 +122,39 @@ public enum DataType { */ SCALED_FLOAT(builder().esType("scaled_float").estimatedSize(Long.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)), + /** + * String fields that are analyzed when the document is received but never + * cut into more than one token. ESQL always loads these after-analysis. + * Generally ESQL uses {@code keyword} fields as raw strings. So things like + * {@code TO_STRING} will make a {@code keyword} field. + */ KEYWORD(builder().esType("keyword").unknownSize().docValues()), + /** + * String fields that are analyzed when the document is received and may be + * cut into more than one token. Generally ESQL only sees {@code text} fields + * when loaded from the index and ESQL will load these fields + * without analysis. The {@code MATCH} operator can be used + * to query these fields with analysis. + */ TEXT(builder().esType("text").unknownSize()), + /** + * Millisecond precision date, stored as a 64-bit signed number. + */ DATETIME(builder().esType("date").typeName("DATETIME").estimatedSize(Long.BYTES).docValues()), + /** + * Nanosecond precision date, stored as a 64-bit signed number. + */ DATE_NANOS(builder().esType("date_nanos").estimatedSize(Long.BYTES).docValues()), /** - * IP addresses, both IPv4 and IPv6, are encoded using 16 bytes. + * IP addresses. IPv4 address are always + * embedded + * in IPv6. These flow through the compute engine as fixed length, 16 byte + * {@link BytesRef}s. */ IP(builder().esType("ip").estimatedSize(16).docValues()), + /** + * A version encoded in a way that sorts using semver. + */ // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer VERSION(builder().esType("version").estimatedSize(15).docValues()), OBJECT(builder().esType("object").unknownSize()), From 929dd4c3c1cbba689690d4c3da990e291ffeabb8 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 9 Aug 2024 20:55:56 +0200 Subject: [PATCH 31/37] Make TestCluster#wipe run steps in parallel where possible (#111744) This is one of the most expensive methods in our tests. We don't have to execute the steps sequentially here, so we can speed things up considerably by only making things that need to wait for each other wait. Also I believe this does add a certain degree of extra coverage on the network layer as we have very few tests that run parallel requests, so we get to stress the allocation logic and such a little extra here. --- .../datastreams/DeleteDataStreamAction.java | 2 +- .../org/elasticsearch/test/TestCluster.java | 322 ++++++++++-------- 2 files changed, 190 insertions(+), 134 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java index 5262c07b6a5d1..1a62e347012fe 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java @@ -113,7 +113,7 @@ public IndicesOptions indicesOptions() { return indicesOptions; } - public IndicesRequest indicesOptions(IndicesOptions options) { + public Request indicesOptions(IndicesOptions options) { this.indicesOptions = options; return this; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java index 5ff73a5ea3286..67ff67ee6fe05 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComponentTemplateAction; @@ -19,25 +20,26 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Random; import java.util.Set; -import java.util.concurrent.TimeUnit; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.ESTestCase.safeAwait; /** * Base test cluster that exposes the basis to run tests against any elasticsearch cluster, whose layout @@ -69,14 +71,99 @@ public void beforeTest(Random randomGenerator) throws IOException, InterruptedEx * Wipes any data that a test can leave behind: indices, templates (except exclude templates) and repositories */ public void wipe(Set excludeTemplates) { - // First delete data streams, because composable index templates can't be deleted if these templates are still used by data streams. - wipeAllDataStreams(); - wipeAllComposableIndexTemplates(excludeTemplates); - wipeAllComponentTemplates(excludeTemplates); - - wipeIndices("_all"); - wipeAllTemplates(excludeTemplates); - wipeRepositories(); + if (size() == 0) { + return; + } + safeAwait((ActionListener done) -> { + try (RefCountingListener listeners = new RefCountingListener(done)) { + wipeAllTemplates(excludeTemplates, listeners); + // First delete data streams, because composable index templates can't be deleted if these templates are still used by data + // streams. + SubscribableListener + + .newForked( + l -> client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(ESTestCase.TEST_REQUEST_TIMEOUT, "*").indicesOptions( + IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN + ), + l.delegateResponse((ll, e) -> { + // Ignore if action isn't registered, because data streams is a module and + // if the delete action isn't registered then there no data streams to delete. + if (e.getMessage().startsWith("failed to find action") == false) { + ll.onFailure(e); + } else { + ll.onResponse(AcknowledgedResponse.TRUE); + } + }) + ) + ) + .andThenAccept(ElasticsearchAssertions::assertAcked) + .andThenAccept(v -> { + SubscribableListener.newForked(ll -> wipeIndicesAsync(new String[] { "_all" }, ll)) + .andThen(this::wipeRepositories) + .addListener(listeners.acquire()); + + deleteTemplates(excludeTemplates, listeners.acquire()); + }) + .addListener(listeners.acquire()); + } + }); + } + + private void deleteTemplates(Set excludeTemplates, ActionListener listener) { + final SubscribableListener getComposableTemplates = SubscribableListener.newForked( + l -> client().execute(GetComposableIndexTemplateAction.INSTANCE, new GetComposableIndexTemplateAction.Request("*"), l) + ); + + final SubscribableListener getComponentTemplates = SubscribableListener.newForked( + l -> client().execute(GetComponentTemplateAction.INSTANCE, new GetComponentTemplateAction.Request("*"), l) + ); + + SubscribableListener + + // dummy start step for symmetry + .newSucceeded(null) + + // delete composable templates + .andThen(getComposableTemplates::addListener) + .andThen((l, r) -> { + var templates = r.indexTemplates() + .keySet() + .stream() + .filter(template -> excludeTemplates.contains(template) == false) + .toArray(String[]::new); + if (templates.length == 0) { + l.onResponse(AcknowledgedResponse.TRUE); + } else { + var request = new TransportDeleteComposableIndexTemplateAction.Request(templates); + client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, request, l); + } + }) + .andThenAccept(ElasticsearchAssertions::assertAcked) + + // then delete component templates + .andThen(getComponentTemplates::addListener) + .andThen((l, response) -> { + var componentTemplates = response.getComponentTemplates() + .keySet() + .stream() + .filter(template -> excludeTemplates.contains(template) == false) + .toArray(String[]::new); + if (componentTemplates.length == 0) { + l.onResponse(AcknowledgedResponse.TRUE); + } else { + client().execute( + TransportDeleteComponentTemplateAction.TYPE, + new TransportDeleteComponentTemplateAction.Request(componentTemplates), + l + ); + } + }) + .andThenAccept(ElasticsearchAssertions::assertAcked) + + // and finish + .addListener(listener); } /** @@ -136,52 +223,88 @@ public void assertAfterTest() throws Exception { * all indices are removed. */ public void wipeIndices(String... indices) { + safeAwait((ActionListener l) -> wipeIndicesAsync(indices, l)); + } + + private void wipeIndicesAsync(String[] indices, ActionListener listener) { assert indices != null && indices.length > 0; - if (size() > 0) { - try { - // include wiping hidden indices! - assertAcked( - client().admin() - .indices() - .prepareDelete(indices) - .setIndicesOptions(IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false)) - ); - } catch (IndexNotFoundException e) { - // ignore - } catch (IllegalArgumentException e) { - // Happens if `action.destructive_requires_name` is set to true - // which is the case in the CloseIndexDisableCloseAllTests - if ("_all".equals(indices[0])) { - ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get(); - ArrayList concreteIndices = new ArrayList<>(); - for (IndexMetadata indexMetadata : clusterStateResponse.getState().metadata()) { - concreteIndices.add(indexMetadata.getIndex().getName()); - } - if (concreteIndices.isEmpty() == false) { - assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(new String[0]))); - } - } + SubscribableListener + + .newForked( + l -> client().admin() + .indices() + .prepareDelete(indices) + .setIndicesOptions( + // include wiping hidden indices! + IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false) + ) + .execute(l.delegateResponse((ll, exception) -> handleWipeIndicesFailure(exception, "_all".equals(indices[0]), ll))) + ) + .andThenAccept(ElasticsearchAssertions::assertAcked) + .addListener(listener); + } + + private void handleWipeIndicesFailure(Exception exception, boolean wipingAllIndices, ActionListener listener) { + Throwable unwrapped = ExceptionsHelper.unwrap(exception, IndexNotFoundException.class, IllegalArgumentException.class); + if (unwrapped instanceof IndexNotFoundException) { + // ignore + listener.onResponse(AcknowledgedResponse.TRUE); + } else if (unwrapped instanceof IllegalArgumentException) { + // Happens if `action.destructive_requires_name` is set to true + // which is the case in the CloseIndexDisableCloseAllTests + if (wipingAllIndices) { + SubscribableListener + + .newForked(l -> client().admin().cluster().prepareState().execute(l)) + .andThen((l, clusterStateResponse) -> { + ArrayList concreteIndices = new ArrayList<>(); + for (IndexMetadata indexMetadata : clusterStateResponse.getState().metadata()) { + concreteIndices.add(indexMetadata.getIndex().getName()); + } + if (concreteIndices.isEmpty() == false) { + client().admin().indices().prepareDelete(concreteIndices.toArray(Strings.EMPTY_ARRAY)).execute(l); + } else { + l.onResponse(AcknowledgedResponse.TRUE); + } + }) + .addListener(listener); + } else { + // TODO: this is clearly wrong but at least + // org.elasticsearch.xpack.watcher.test.integration.BootStrapTests.testTriggeredWatchLoading depends on this + // quietly passing when it tries to delete an alias instead of its backing indices + listener.onResponse(AcknowledgedResponse.TRUE); } + } else { + listener.onFailure(exception); } } /** * Removes all templates, except the templates defined in the exclude */ - public void wipeAllTemplates(Set exclude) { - if (size() > 0) { - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get(); - for (IndexTemplateMetadata indexTemplate : response.getIndexTemplates()) { - if (exclude.contains(indexTemplate.getName())) { - continue; - } - try { - client().admin().indices().prepareDeleteTemplate(indexTemplate.getName()).get(); - } catch (IndexTemplateMissingException e) { - // ignore + private void wipeAllTemplates(Set exclude, RefCountingListener listeners) { + SubscribableListener + + .newForked(l -> client().admin().indices().prepareGetTemplates().execute(l)) + .andThenAccept(response -> { + for (IndexTemplateMetadata indexTemplate : response.getIndexTemplates()) { + if (exclude.contains(indexTemplate.getName())) { + continue; + } + client().admin() + .indices() + .prepareDeleteTemplate(indexTemplate.getName()) + .execute(listeners.acquire(ElasticsearchAssertions::assertAcked).delegateResponse((l, e) -> { + if (e instanceof IndexTemplateMissingException) { + // ignore + l.onResponse(AcknowledgedResponse.TRUE); + } else { + l.onFailure(e); + } + })); } - } - } + }) + .addListener(listeners.acquire()); } /** @@ -207,91 +330,24 @@ public void wipeTemplates(String... templates) { /** * Deletes repositories, supports wildcard notation. */ - public void wipeRepositories(String... repositories) { - if (size() > 0) { - // if nothing is provided, delete all - if (repositories.length == 0) { - repositories = new String[] { "*" }; - } - final var future = new PlainActionFuture(); - try (var listeners = new RefCountingListener(future)) { - for (String repository : repositories) { - ActionListener.run( - listeners.acquire(), - l -> client().admin() - .cluster() - .prepareDeleteRepository(ESTestCase.TEST_REQUEST_TIMEOUT, ESTestCase.TEST_REQUEST_TIMEOUT, repository) - .execute(new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - l.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - if (e instanceof RepositoryMissingException) { - // ignore - l.onResponse(null); - } else { - l.onFailure(e); - } - } - }) - ); - } - } - future.actionGet(30, TimeUnit.SECONDS); - } - } - - public void wipeAllDataStreams() { - if (size() > 0) { - var request = new DeleteDataStreamAction.Request(ESTestCase.TEST_REQUEST_TIMEOUT, "*"); - request.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); - try { - assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, request).actionGet()); - } catch (IllegalStateException e) { - // Ignore if action isn't registered, because data streams is a module and - // if the delete action isn't registered then there no data streams to delete. - if (e.getMessage().startsWith("failed to find action") == false) { - throw e; - } - } - } - } - - public void wipeAllComposableIndexTemplates(Set excludeTemplates) { - if (size() > 0) { - var templates = client().execute(GetComposableIndexTemplateAction.INSTANCE, new GetComposableIndexTemplateAction.Request("*")) - .actionGet() - .indexTemplates() - .keySet() - .stream() - .filter(template -> excludeTemplates.contains(template) == false) - .toArray(String[]::new); - - if (templates.length != 0) { - var request = new TransportDeleteComposableIndexTemplateAction.Request(templates); - assertAcked(client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, request).actionGet()); - } - } - } - - public void wipeAllComponentTemplates(Set excludeTemplates) { - if (size() > 0) { - var templates = client().execute(GetComponentTemplateAction.INSTANCE, new GetComponentTemplateAction.Request("*")) - .actionGet() - .getComponentTemplates() - .keySet() - .stream() - .filter(template -> excludeTemplates.contains(template) == false) - .toArray(String[]::new); - - if (templates.length != 0) { - var request = new TransportDeleteComponentTemplateAction.Request(templates); - assertAcked(client().execute(TransportDeleteComponentTemplateAction.TYPE, request).actionGet()); - } - } + private void wipeRepositories(ActionListener listener) { + SubscribableListener + + .newForked( + l -> client().admin() + .cluster() + .prepareDeleteRepository(ESTestCase.TEST_REQUEST_TIMEOUT, ESTestCase.TEST_REQUEST_TIMEOUT, "*") + .execute(l.delegateResponse((ll, e) -> { + if (e instanceof RepositoryMissingException) { + // ignore + l.onResponse(AcknowledgedResponse.TRUE); + } else { + l.onFailure(e); + } + })) + ) + .andThenAccept(ElasticsearchAssertions::assertAcked) + .addListener(listener); } /** From fd916c221ae51b621dc5eb4d93b8f5a10bbc6d49 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 10 Aug 2024 06:39:24 +1000 Subject: [PATCH 32/37] Mute org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT #111765 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e439759d75dab..734342cf896d2 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -125,6 +125,8 @@ tests: - class: org.elasticsearch.tdigest.ComparisonTests method: testSparseGaussianDistribution issue: https://github.com/elastic/elasticsearch/issues/111721 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/111765 # Examples: # From 59cf661e2ea5a3d344f1adce4c37b5afacdeafa4 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 12 Aug 2024 09:02:33 +0900 Subject: [PATCH 33/37] Speed up dense/sparse vector stats (#111729) This change ensures that we don't try to compute stats on mappings that don't have dense or sparse vector fields. We don't need to go through all the fields on every segment, instead we can extract the vector fields upfront and limit the work to only indices that define these types. Closes #111715 --- docs/changelog/111729.yaml | 6 ++ .../elasticsearch/index/engine/Engine.java | 82 +++++++++++-------- .../elasticsearch/index/shard/IndexShard.java | 3 +- .../index/engine/frozen/FrozenEngine.java | 14 ++-- 4 files changed, 61 insertions(+), 44 deletions(-) create mode 100644 docs/changelog/111729.yaml diff --git a/docs/changelog/111729.yaml b/docs/changelog/111729.yaml new file mode 100644 index 0000000000000..c75c14a997da9 --- /dev/null +++ b/docs/changelog/111729.yaml @@ -0,0 +1,6 @@ +pr: 111729 +summary: Speed up dense/sparse vector stats +area: Vector Search +type: bug +issues: + - 111715 diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 6f4511483126f..b07132eea75e8 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -61,7 +61,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.DocumentParser; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.Mapper; @@ -69,6 +68,7 @@ import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.seqno.SeqNoStats; @@ -242,19 +242,32 @@ protected final DocsStats docsStats(IndexReader indexReader) { /** * Returns the {@link DenseVectorStats} for this engine */ - public DenseVectorStats denseVectorStats() { + public DenseVectorStats denseVectorStats(MappingLookup mappingLookup) { + if (mappingLookup == null) { + return new DenseVectorStats(0); + } + + List fields = new ArrayList<>(); + for (Mapper mapper : mappingLookup.fieldMappers()) { + if (mapper instanceof DenseVectorFieldMapper) { + fields.add(mapper.fullPath()); + } + } + if (fields.isEmpty()) { + return new DenseVectorStats(0); + } try (Searcher searcher = acquireSearcher(DOC_STATS_SOURCE, SearcherScope.INTERNAL)) { - return denseVectorStats(searcher.getIndexReader()); + return denseVectorStats(searcher.getIndexReader(), fields); } } - protected final DenseVectorStats denseVectorStats(IndexReader indexReader) { + protected final DenseVectorStats denseVectorStats(IndexReader indexReader, List fields) { long valueCount = 0; // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause // the next scheduled refresh to go through and refresh the stats as well for (LeafReaderContext readerContext : indexReader.leaves()) { try { - valueCount += getDenseVectorValueCount(readerContext.reader()); + valueCount += getDenseVectorValueCount(readerContext.reader(), fields); } catch (IOException e) { logger.trace(() -> "failed to get dense vector stats for [" + readerContext + "]", e); } @@ -262,9 +275,10 @@ protected final DenseVectorStats denseVectorStats(IndexReader indexReader) { return new DenseVectorStats(valueCount); } - private long getDenseVectorValueCount(final LeafReader atomicReader) throws IOException { + private long getDenseVectorValueCount(final LeafReader atomicReader, List fields) throws IOException { long count = 0; - for (FieldInfo info : atomicReader.getFieldInfos()) { + for (var field : fields) { + var info = atomicReader.getFieldInfos().fieldInfo(field); if (info.getVectorDimension() > 0) { switch (info.getVectorEncoding()) { case FLOAT32 -> { @@ -285,23 +299,31 @@ private long getDenseVectorValueCount(final LeafReader atomicReader) throws IOEx * Returns the {@link SparseVectorStats} for this engine */ public SparseVectorStats sparseVectorStats(MappingLookup mappingLookup) { + if (mappingLookup == null) { + return new SparseVectorStats(0); + } + List fields = new ArrayList<>(); + for (Mapper mapper : mappingLookup.fieldMappers()) { + if (mapper instanceof SparseVectorFieldMapper) { + fields.add(new BytesRef(mapper.fullPath())); + } + } + if (fields.isEmpty()) { + return new SparseVectorStats(0); + } + Collections.sort(fields); try (Searcher searcher = acquireSearcher(DOC_STATS_SOURCE, SearcherScope.INTERNAL)) { - return sparseVectorStats(searcher.getIndexReader(), mappingLookup); + return sparseVectorStats(searcher.getIndexReader(), fields); } } - protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, MappingLookup mappingLookup) { + protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, List fields) { long valueCount = 0; - - if (mappingLookup == null) { - return new SparseVectorStats(valueCount); - } - // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause // the next scheduled refresh to go through and refresh the stats as well for (LeafReaderContext readerContext : indexReader.leaves()) { try { - valueCount += getSparseVectorValueCount(readerContext.reader(), mappingLookup); + valueCount += getSparseVectorValueCount(readerContext.reader(), fields); } catch (IOException e) { logger.trace(() -> "failed to get sparse vector stats for [" + readerContext + "]", e); } @@ -309,28 +331,16 @@ protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, Map return new SparseVectorStats(valueCount); } - private long getSparseVectorValueCount(final LeafReader atomicReader, MappingLookup mappingLookup) throws IOException { + private long getSparseVectorValueCount(final LeafReader atomicReader, List fields) throws IOException { long count = 0; - - Map mappers = new HashMap<>(); - for (Mapper mapper : mappingLookup.fieldMappers()) { - if (mapper instanceof FieldMapper fieldMapper) { - if (fieldMapper.fieldType() instanceof SparseVectorFieldMapper.SparseVectorFieldType) { - mappers.put(fieldMapper.fullPath(), fieldMapper); - } - } - } - - for (FieldInfo info : atomicReader.getFieldInfos()) { - String name = info.name; - if (mappers.containsKey(name)) { - Terms terms = atomicReader.terms(FieldNamesFieldMapper.NAME); - if (terms != null) { - TermsEnum termsEnum = terms.iterator(); - if (termsEnum.seekExact(new BytesRef(name))) { - count += termsEnum.docFreq(); - } - } + Terms terms = atomicReader.terms(FieldNamesFieldMapper.NAME); + if (terms == null) { + return count; + } + TermsEnum termsEnum = terms.iterator(); + for (var fieldName : fields) { + if (termsEnum.seekExact(fieldName)) { + count += termsEnum.docFreq(); } } return count; diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index bc0d9ce2a84d7..b7d1beb4d1e06 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1428,7 +1428,8 @@ public CompletionStats completionStats(String... fields) { public DenseVectorStats denseVectorStats() { readAllowed(); - return getEngine().denseVectorStats(); + MappingLookup mappingLookup = mapperService != null ? mapperService.mappingLookup() : null; + return getEngine().denseVectorStats(mappingLookup); } public SparseVectorStats sparseVectorStats() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java index 0a13aab82aced..3b242ca94ac61 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java @@ -63,8 +63,6 @@ public final class FrozenEngine extends ReadOnlyEngine { ); private final SegmentsStats segmentsStats; private final DocsStats docsStats; - private final DenseVectorStats denseVectorStats; - private final SparseVectorStats sparseVectorStats; private volatile ElasticsearchDirectoryReader lastOpenedReader; private final ElasticsearchDirectoryReader canMatchReader; private final Object cacheIdentity = new Object(); @@ -95,8 +93,6 @@ public FrozenEngine( fillSegmentStats(segmentReader, true, segmentsStats); } this.docsStats = docsStats(reader); - this.denseVectorStats = denseVectorStats(reader); - this.sparseVectorStats = sparseVectorStats(reader, null); canMatchReader = ElasticsearchDirectoryReader.wrap( new RewriteCachingDirectoryReader(directory, reader.leaves(), null), config.getShardId() @@ -334,13 +330,17 @@ public DocsStats docStats() { } @Override - public DenseVectorStats denseVectorStats() { - return denseVectorStats; + public DenseVectorStats denseVectorStats(MappingLookup mappingLookup) { + // We could cache the result on first call but dense vectors on frozen tier + // are very unlikely, so we just don't count them in the stats. + return new DenseVectorStats(0); } @Override public SparseVectorStats sparseVectorStats(MappingLookup mappingLookup) { - return sparseVectorStats; + // We could cache the result on first call but sparse vectors on frozen tier + // are very unlikely, so we just don't count them in the stats. + return new SparseVectorStats(0); } synchronized boolean isReaderOpen() { From 7b7c310ea5ea5aa6f306a1e740a1eec9532586f1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 12 Aug 2024 09:38:02 +0200 Subject: [PATCH 34/37] [DOCS] Fix elasticsearch-py helpers page link (#111789) --- .../search/search-your-data/paginate-search-results.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/search/search-your-data/paginate-search-results.asciidoc b/docs/reference/search/search-your-data/paginate-search-results.asciidoc index 3ff49b27f6f5d..edd1546dd0854 100644 --- a/docs/reference/search/search-your-data/paginate-search-results.asciidoc +++ b/docs/reference/search/search-your-data/paginate-search-results.asciidoc @@ -362,7 +362,7 @@ Perl:: Python:: - See https://elasticsearch-py.readthedocs.io/en/[elasticsearch.helpers.*] + See https://elasticsearch-py.readthedocs.io/en/stable/helpers.html[elasticsearch.helpers.*] JavaScript:: From 364bba2e6b975d9247b54e91b4ced5a8b5abe6bb Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Mon, 12 Aug 2024 10:09:14 +0200 Subject: [PATCH 35/37] [cache] Support async RangeMissingHandler callbacks (#111340) Change `fillCacheRange` method to accept a completion listener that must be called by `RangeMissingHandler` implementations when they finish fetching data. By doing so, we support asynchronously fetching the data from a third party storage. We also support asynchronous `SourceInputStreamFactory` for reading gaps from the storage. --- .../shared/SharedBlobCacheService.java | 101 +++++--- .../shared/SharedBlobCacheServiceTests.java | 216 ++++++++++++------ .../store/input/FrozenIndexInput.java | 59 ++--- 3 files changed, 253 insertions(+), 123 deletions(-) diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index 3d95db72e269d..28a5eb164d049 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -646,13 +646,14 @@ private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, int wri // no need to allocate a new capturing lambda if the offset isn't adjusted return writer; } - return (channel, channelPos, streamFactory, relativePos, len, progressUpdater) -> writer.fillCacheRange( + return (channel, channelPos, streamFactory, relativePos, len, progressUpdater, completionListener) -> writer.fillCacheRange( channel, channelPos, streamFactory, relativePos - writeOffset, len, - progressUpdater + progressUpdater, + completionListener ); } @@ -987,16 +988,17 @@ void populateAndRead( executor.execute(fillGapRunnable(gap, writer, null, refs.acquireListener())); } } else { - final List gapFillingTasks = gaps.stream() - .map(gap -> fillGapRunnable(gap, writer, streamFactory, refs.acquireListener())) - .toList(); - executor.execute(() -> { - try (streamFactory) { + var gapFillingListener = refs.acquireListener(); + try (var gfRefs = new RefCountingRunnable(ActionRunnable.run(gapFillingListener, streamFactory::close))) { + final List gapFillingTasks = gaps.stream() + .map(gap -> fillGapRunnable(gap, writer, streamFactory, gfRefs.acquireListener())) + .toList(); + executor.execute(() -> { // Fill the gaps in order. If a gap fails to fill for whatever reason, the task for filling the next // gap will still be executed. gapFillingTasks.forEach(Runnable::run); - } - }); + }); + } } } } @@ -1005,13 +1007,13 @@ void populateAndRead( } } - private AbstractRunnable fillGapRunnable( + private Runnable fillGapRunnable( SparseFileTracker.Gap gap, RangeMissingHandler writer, @Nullable SourceInputStreamFactory streamFactory, ActionListener listener ) { - return ActionRunnable.run(listener.delegateResponse((l, e) -> failGapAndListener(gap, l, e)), () -> { + return () -> ActionListener.run(listener, l -> { var ioRef = io; assert regionOwners.get(ioRef) == CacheFileRegion.this; assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this; @@ -1022,10 +1024,15 @@ private AbstractRunnable fillGapRunnable( streamFactory, start, Math.toIntExact(gap.end() - start), - progress -> gap.onProgress(start + progress) + progress -> gap.onProgress(start + progress), + l.map(unused -> { + assert regionOwners.get(ioRef) == CacheFileRegion.this; + assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this; + writeCount.increment(); + gap.onCompletion(); + return null; + }).delegateResponse((delegate, e) -> failGapAndListener(gap, delegate, e)) ); - writeCount.increment(); - gap.onCompletion(); }); } @@ -1113,12 +1120,23 @@ public void fillCacheRange( SourceInputStreamFactory streamFactory, int relativePos, int length, - IntConsumer progressUpdater + IntConsumer progressUpdater, + ActionListener completionListener ) throws IOException { - writer.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater); - var elapsedTime = TimeUnit.NANOSECONDS.toMillis(relativeTimeInNanosSupplier.getAsLong() - startTime); - SharedBlobCacheService.this.blobCacheMetrics.getCacheMissLoadTimes().record(elapsedTime); - SharedBlobCacheService.this.blobCacheMetrics.getCacheMissCounter().increment(); + writer.fillCacheRange( + channel, + channelPos, + streamFactory, + relativePos, + length, + progressUpdater, + completionListener.map(unused -> { + var elapsedTime = TimeUnit.NANOSECONDS.toMillis(relativeTimeInNanosSupplier.getAsLong() - startTime); + blobCacheMetrics.getCacheMissLoadTimes().record(elapsedTime); + blobCacheMetrics.getCacheMissCounter().increment(); + return null; + }) + ); } }; if (rangeToRead.isEmpty()) { @@ -1211,9 +1229,18 @@ public void fillCacheRange( SourceInputStreamFactory streamFactory, int relativePos, int len, - IntConsumer progressUpdater + IntConsumer progressUpdater, + ActionListener completionListener ) throws IOException { - delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos - writeOffset, len, progressUpdater); + delegate.fillCacheRange( + channel, + channelPos, + streamFactory, + relativePos - writeOffset, + len, + progressUpdater, + completionListener + ); } }; } @@ -1226,14 +1253,25 @@ public void fillCacheRange( SourceInputStreamFactory streamFactory, int relativePos, int len, - IntConsumer progressUpdater + IntConsumer progressUpdater, + ActionListener completionListener ) throws IOException { assert assertValidRegionAndLength(fileRegion, channelPos, len); - delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, len, progressUpdater); - assert regionOwners.get(fileRegion.io) == fileRegion - : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]"; + delegate.fillCacheRange( + channel, + channelPos, + streamFactory, + relativePos, + len, + progressUpdater, + Assertions.ENABLED ? ActionListener.runBefore(completionListener, () -> { + assert regionOwners.get(fileRegion.io) == fileRegion + : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]"; + }) : completionListener + ); } }; + } return adjustedWriter; } @@ -1320,6 +1358,7 @@ default SourceInputStreamFactory sharedInputStreamFactory(List completionListener ) throws IOException; } @@ -1339,9 +1379,9 @@ public interface SourceInputStreamFactory extends Releasable { /** * Create the input stream at the specified position. * @param relativePos the relative position in the remote storage to read from. - * @return the input stream ready to be read from. + * @param listener listener for the input stream ready to be read from. */ - InputStream create(int relativePos) throws IOException; + void create(int relativePos, ActionListener listener) throws IOException; } private abstract static class DelegatingRangeMissingHandler implements RangeMissingHandler { @@ -1363,9 +1403,10 @@ public void fillCacheRange( SourceInputStreamFactory streamFactory, int relativePos, int length, - IntConsumer progressUpdater + IntConsumer progressUpdater, + ActionListener completionListener ) throws IOException { - delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater); + delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener); } } diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java index e477673c90d6d..6c49b50c06e82 100644 --- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java +++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.StoppableExecutorServiceWrapper; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.TestEnvironment; @@ -72,6 +73,13 @@ private static long size(long numPages) { return numPages * SharedBytes.PAGE_SIZE; } + private static void completeWith(ActionListener listener, CheckedRunnable runnable) { + ActionListener.completeWith(listener, () -> { + runnable.run(); + return null; + }); + } + public void testBasicEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") @@ -115,7 +123,10 @@ public void testBasicEviction() throws IOException { ByteRange.of(0L, 1L), ByteRange.of(0L, 1L), (channel, channelPos, relativePos, length) -> 1, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> progressUpdater.accept(length) + ), taskQueue.getThreadPool().generic(), bytesReadFuture ); @@ -552,11 +563,14 @@ public void execute(Runnable command) { cacheService.maybeFetchFullEntry( cacheKey, size, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - assert streamFactory == null : streamFactory; - bytesRead.addAndGet(-length); - progressUpdater.accept(length); - }, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + assert streamFactory == null : streamFactory; + bytesRead.addAndGet(-length); + progressUpdater.accept(length); + } + ), bulkExecutor, future ); @@ -570,9 +584,15 @@ public void execute(Runnable command) { // a download that would use up all regions should not run final var cacheKey = generateCacheKey(); assertEquals(2, cacheService.freeRegionCount()); - var configured = cacheService.maybeFetchFullEntry(cacheKey, size(500), (ch, chPos, streamFactory, relPos, len, update) -> { - throw new AssertionError("Should never reach here"); - }, bulkExecutor, ActionListener.noop()); + var configured = cacheService.maybeFetchFullEntry( + cacheKey, + size(500), + (ch, chPos, streamFactory, relPos, len, update, completionListener) -> completeWith(completionListener, () -> { + throw new AssertionError("Should never reach here"); + }), + bulkExecutor, + ActionListener.noop() + ); assertFalse(configured); assertEquals(2, cacheService.freeRegionCount()); } @@ -613,9 +633,14 @@ public void testFetchFullCacheEntryConcurrently() throws Exception { (ActionListener listener) -> cacheService.maybeFetchFullEntry( cacheKey, size, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept( - length - ), + ( + channel, + channelPos, + streamFactory, + relativePos, + length, + progressUpdater, + completionListener) -> completeWith(completionListener, () -> progressUpdater.accept(length)), bulkExecutor, listener ) @@ -859,7 +884,10 @@ public void testMaybeEvictLeastUsed() throws Exception { var entry = cacheService.get(cacheKey, regionSize, 0); entry.populate( ByteRange.of(0L, regionSize), - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> progressUpdater.accept(length) + ), taskQueue.getThreadPool().generic(), ActionListener.noop() ); @@ -954,11 +982,14 @@ public void execute(Runnable command) { cacheKey, 0, blobLength, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - assert streamFactory == null : streamFactory; - bytesRead.addAndGet(length); - progressUpdater.accept(length); - }, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + assert streamFactory == null : streamFactory; + bytesRead.addAndGet(length); + progressUpdater.accept(length); + } + ), bulkExecutor, future ); @@ -985,11 +1016,14 @@ public void execute(Runnable command) { cacheKey, region, blobLength, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - assert streamFactory == null : streamFactory; - bytesRead.addAndGet(length); - progressUpdater.accept(length); - }, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + assert streamFactory == null : streamFactory; + bytesRead.addAndGet(length); + progressUpdater.accept(length); + } + ), bulkExecutor, listener ); @@ -1010,9 +1044,12 @@ public void execute(Runnable command) { cacheKey, randomIntBetween(0, 10), randomLongBetween(1L, regionSize), - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - throw new AssertionError("should not be executed"); - }, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + throw new AssertionError("should not be executed"); + } + ), bulkExecutor, future ); @@ -1032,11 +1069,14 @@ public void execute(Runnable command) { cacheKey, 0, blobLength, - (channel, channelPos, ignore, relativePos, length, progressUpdater) -> { - assert ignore == null : ignore; - bytesRead.addAndGet(length); - progressUpdater.accept(length); - }, + (channel, channelPos, ignore, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + assert ignore == null : ignore; + bytesRead.addAndGet(length); + progressUpdater.accept(length); + } + ), bulkExecutor, future ); @@ -1110,12 +1150,15 @@ public void execute(Runnable command) { region, range, blobLength, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - assertThat(range.start() + relativePos, equalTo(cacheService.getRegionStart(region) + regionRange.start())); - assertThat(channelPos, equalTo(Math.toIntExact(regionRange.start()))); - assertThat(length, equalTo(Math.toIntExact(regionRange.length()))); - bytesCopied.addAndGet(length); - }, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + assertThat(range.start() + relativePos, equalTo(cacheService.getRegionStart(region) + regionRange.start())); + assertThat(channelPos, equalTo(Math.toIntExact(regionRange.start()))); + assertThat(length, equalTo(Math.toIntExact(regionRange.length()))); + bytesCopied.addAndGet(length); + } + ), bulkExecutor, future ); @@ -1150,7 +1193,10 @@ public void execute(Runnable command) { region, ByteRange.of(0L, blobLength), blobLength, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> bytesCopied.addAndGet(length) + ), bulkExecutor, listener ); @@ -1173,9 +1219,12 @@ public void execute(Runnable command) { randomIntBetween(0, 10), ByteRange.of(0L, blobLength), blobLength, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - throw new AssertionError("should not be executed"); - }, + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + throw new AssertionError("should not be executed"); + } + ), bulkExecutor, future ); @@ -1196,7 +1245,10 @@ public void execute(Runnable command) { 0, ByteRange.of(0L, blobLength), blobLength, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> bytesCopied.addAndGet(length) + ), bulkExecutor, future ); @@ -1237,10 +1289,18 @@ public void testPopulate() throws Exception { var entry = cacheService.get(cacheKey, blobLength, 0); AtomicLong bytesWritten = new AtomicLong(0L); final PlainActionFuture future1 = new PlainActionFuture<>(); - entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - bytesWritten.addAndGet(length); - progressUpdater.accept(length); - }, taskQueue.getThreadPool().generic(), future1); + entry.populate( + ByteRange.of(0, regionSize - 1), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + bytesWritten.addAndGet(length); + progressUpdater.accept(length); + } + ), + taskQueue.getThreadPool().generic(), + future1 + ); assertThat(future1.isDone(), is(false)); assertThat(taskQueue.hasRunnableTasks(), is(true)); @@ -1248,18 +1308,34 @@ public void testPopulate() throws Exception { // start populating the second region entry = cacheService.get(cacheKey, blobLength, 1); final PlainActionFuture future2 = new PlainActionFuture<>(); - entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - bytesWritten.addAndGet(length); - progressUpdater.accept(length); - }, taskQueue.getThreadPool().generic(), future2); + entry.populate( + ByteRange.of(0, regionSize - 1), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + bytesWritten.addAndGet(length); + progressUpdater.accept(length); + } + ), + taskQueue.getThreadPool().generic(), + future2 + ); // start populating again the first region, listener should be called immediately entry = cacheService.get(cacheKey, blobLength, 0); final PlainActionFuture future3 = new PlainActionFuture<>(); - entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> { - bytesWritten.addAndGet(length); - progressUpdater.accept(length); - }, taskQueue.getThreadPool().generic(), future3); + entry.populate( + ByteRange.of(0, regionSize - 1), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> { + bytesWritten.addAndGet(length); + progressUpdater.accept(length); + } + ), + taskQueue.getThreadPool().generic(), + future3 + ); assertThat(future3.isDone(), is(true)); var written = future3.get(10L, TimeUnit.SECONDS); @@ -1377,7 +1453,10 @@ public void testSharedSourceInputStreamFactory() throws Exception { range, range, (channel, channelPos, relativePos, length) -> length, - (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length), + (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith( + completionListener, + () -> progressUpdater.accept(length) + ), EsExecutors.DIRECT_EXECUTOR_SERVICE, future ); @@ -1394,8 +1473,8 @@ public void testSharedSourceInputStreamFactory() throws Exception { final var factoryClosed = new AtomicBoolean(false); final var dummyStreamFactory = new SourceInputStreamFactory() { @Override - public InputStream create(int relativePos) { - return null; + public void create(int relativePos, ActionListener listener) { + listener.onResponse(null); } @Override @@ -1420,17 +1499,20 @@ public void fillCacheRange( SourceInputStreamFactory streamFactory, int relativePos, int length, - IntConsumer progressUpdater + IntConsumer progressUpdater, + ActionListener completion ) throws IOException { - if (invocationCounter.incrementAndGet() == 1) { - final Thread witness = invocationThread.compareAndExchange(null, Thread.currentThread()); - assertThat(witness, nullValue()); - } else { - assertThat(invocationThread.get(), sameInstance(Thread.currentThread())); - } - assertThat(streamFactory, sameInstance(dummyStreamFactory)); - assertThat(position.getAndSet(relativePos), lessThan(relativePos)); - progressUpdater.accept(length); + completeWith(completion, () -> { + if (invocationCounter.incrementAndGet() == 1) { + final Thread witness = invocationThread.compareAndExchange(null, Thread.currentThread()); + assertThat(witness, nullValue()); + } else { + assertThat(invocationThread.get(), sameInstance(Thread.currentThread())); + } + assertThat(streamFactory, sameInstance(dummyStreamFactory)); + assertThat(position.getAndSet(relativePos), lessThan(relativePos)); + progressUpdater.accept(length); + }); } }; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java index 56efc72f2f6f7..d7cf22a05981f 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.blobcache.BlobCacheUtils; import org.elasticsearch.blobcache.common.ByteBufferReference; import org.elasticsearch.blobcache.common.ByteRange; @@ -146,32 +147,38 @@ private void readWithoutBlobCacheSlow(ByteBuffer b, long position, int length) t final int read = SharedBytes.readCacheFile(channel, pos, relativePos, len, byteBufferReference); stats.addCachedBytesRead(read); return read; - }, (channel, channelPos, streamFactory, relativePos, len, progressUpdater) -> { - assert streamFactory == null : streamFactory; - final long startTimeNanos = stats.currentTimeNanos(); - try (InputStream input = openInputStreamFromBlobStore(rangeToWrite.start() + relativePos, len)) { - assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME); - logger.trace( - "{}: writing channel {} pos {} length {} (details: {})", - fileInfo.physicalName(), - channelPos, - relativePos, - len, - cacheFile - ); - SharedBytes.copyToCacheFileAligned( - channel, - input, - channelPos, - relativePos, - len, - progressUpdater, - writeBuffer.get().clear() - ); - final long endTimeNanos = stats.currentTimeNanos(); - stats.addCachedBytesWritten(len, endTimeNanos - startTimeNanos); - } - }); + }, + (channel, channelPos, streamFactory, relativePos, len, progressUpdater, completionListener) -> ActionListener.completeWith( + completionListener, + () -> { + assert streamFactory == null : streamFactory; + final long startTimeNanos = stats.currentTimeNanos(); + try (InputStream input = openInputStreamFromBlobStore(rangeToWrite.start() + relativePos, len)) { + assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME); + logger.trace( + "{}: writing channel {} pos {} length {} (details: {})", + fileInfo.physicalName(), + channelPos, + relativePos, + len, + cacheFile + ); + SharedBytes.copyToCacheFileAligned( + channel, + input, + channelPos, + relativePos, + len, + progressUpdater, + writeBuffer.get().clear() + ); + final long endTimeNanos = stats.currentTimeNanos(); + stats.addCachedBytesWritten(len, endTimeNanos - startTimeNanos); + return null; + } + } + ) + ); assert bytesRead == length : bytesRead + " vs " + length; byteBufferReference.finish(bytesRead); } finally { From ccfbcd4bee92cc36fe48bc40f96cbdedb95f6a01 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 12 Aug 2024 10:15:47 +0200 Subject: [PATCH 36/37] Parallelise ESIntegTestCase.ensureClusterStateConsistency across multiple nodes (#111769) We can get a non-trivial speedup here but running all the nodes in parallel. This is quite helpful for end-to-end integ test performance since we run this method after effectively every test. --- .../elasticsearch/test/ESIntegTestCase.java | 107 +++++++++++------- .../ml/integration/MlNativeIntegTestCase.java | 42 +------ 2 files changed, 68 insertions(+), 81 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 41afec9aa1120..aad3dcc457241 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -59,6 +59,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; @@ -1235,48 +1236,74 @@ protected void ensureClusterSizeConsistency() { */ protected void ensureClusterStateConsistency() throws IOException { if (cluster() != null && cluster().size() > 0) { - final NamedWriteableRegistry namedWriteableRegistry = cluster().getNamedWriteableRegistry(); - final Client masterClient = client(); - ClusterState masterClusterState = masterClient.admin().cluster().prepareState().all().get().getState(); - byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); - // remove local node reference - masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); - Map masterStateMap = convertToMap(masterClusterState); - int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; - String masterId = masterClusterState.nodes().getMasterNodeId(); - for (Client client : cluster().getClients()) { - ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState(); - byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState); - // remove local node reference - localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry); - final Map localStateMap = convertToMap(localClusterState); - final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; - // Check that the non-master node has the same version of the cluster state as the master and - // that the master node matches the master (otherwise there is no requirement for the cluster state to match) - if (masterClusterState.version() == localClusterState.version() - && masterId.equals(localClusterState.nodes().getMasterNodeId())) { - try { - assertEquals("cluster state UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID()); - // We cannot compare serialization bytes since serialization order of maps is not guaranteed - // but we can compare serialization sizes - they should be the same - assertEquals("cluster state size does not match", masterClusterStateSize, localClusterStateSize); - // Compare JSON serialization - assertNull( - "cluster state JSON serialization does not match", - differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap) - ); - } catch (final AssertionError error) { - logger.error( - "Cluster state from master:\n{}\nLocal cluster state:\n{}", - masterClusterState.toString(), - localClusterState.toString() - ); - throw error; - } - } - } + doEnsureClusterStateConsistency(cluster().getNamedWriteableRegistry()); } + } + protected final void doEnsureClusterStateConsistency(NamedWriteableRegistry namedWriteableRegistry) { + final PlainActionFuture future = new PlainActionFuture<>(); + final List> localStates = new ArrayList<>(cluster().size()); + for (Client client : cluster().getClients()) { + localStates.add(SubscribableListener.newForked(l -> client.admin().cluster().prepareState().all().setLocal(true).execute(l))); + } + try (RefCountingListener refCountingListener = new RefCountingListener(future)) { + SubscribableListener.newForked(l -> client().admin().cluster().prepareState().all().execute(l)) + .andThenAccept(masterStateResponse -> { + byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterStateResponse.getState()); + // remove local node reference + final ClusterState masterClusterState = ClusterState.Builder.fromBytes( + masterClusterStateBytes, + null, + namedWriteableRegistry + ); + Map masterStateMap = convertToMap(masterClusterState); + int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; + String masterId = masterClusterState.nodes().getMasterNodeId(); + for (SubscribableListener localStateListener : localStates) { + localStateListener.andThenAccept(localClusterStateResponse -> { + byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterStateResponse.getState()); + // remove local node reference + final ClusterState localClusterState = ClusterState.Builder.fromBytes( + localClusterStateBytes, + null, + namedWriteableRegistry + ); + final Map localStateMap = convertToMap(localClusterState); + final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; + // Check that the non-master node has the same version of the cluster state as the master and + // that the master node matches the master (otherwise there is no requirement for the cluster state to + // match) + if (masterClusterState.version() == localClusterState.version() + && masterId.equals(localClusterState.nodes().getMasterNodeId())) { + try { + assertEquals( + "cluster state UUID does not match", + masterClusterState.stateUUID(), + localClusterState.stateUUID() + ); + // We cannot compare serialization bytes since serialization order of maps is not guaranteed + // but we can compare serialization sizes - they should be the same + assertEquals("cluster state size does not match", masterClusterStateSize, localClusterStateSize); + // Compare JSON serialization + assertNull( + "cluster state JSON serialization does not match", + differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap) + ); + } catch (final AssertionError error) { + logger.error( + "Cluster state from master:\n{}\nLocal cluster state:\n{}", + masterClusterState.toString(), + localClusterState.toString() + ); + throw error; + } + } + }).addListener(refCountingListener.acquire()); + } + }) + .addListener(refCountingListener.acquire()); + } + safeGet(future); } protected void ensureClusterStateCanBeReadByNodeTool() throws IOException { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 3d918dd414a77..ca5ecd80a83bb 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -118,8 +118,6 @@ import java.util.Set; import java.util.function.Function; -import static org.elasticsearch.test.XContentTestUtils.convertToMap; -import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.monitoring.MonitoringService.ELASTICSEARCH_COLLECTION_ENABLED; @@ -408,45 +406,7 @@ protected void ensureClusterStateConsistency() throws IOException { entries.add( new NamedWriteableRegistry.Entry(AutoscalingDeciderResult.Reason.class, MlScalingReason.NAME, MlScalingReason::new) ); - final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries); - ClusterState masterClusterState = clusterAdmin().prepareState().all().get().getState(); - byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); - // remove local node reference - masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); - Map masterStateMap = convertToMap(masterClusterState); - int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; - String masterId = masterClusterState.nodes().getMasterNodeId(); - for (Client client : cluster().getClients()) { - ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState(); - byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState); - // remove local node reference - localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry); - final Map localStateMap = convertToMap(localClusterState); - final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; - // Check that the non-master node has the same version of the cluster state as the master and - // that the master node matches the master (otherwise there is no requirement for the cluster state to match) - if (masterClusterState.version() == localClusterState.version() - && masterId.equals(localClusterState.nodes().getMasterNodeId())) { - try { - assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID()); - // We cannot compare serialization bytes since serialization order of maps is not guaranteed - // but we can compare serialization sizes - they should be the same - assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize); - // Compare JSON serialization - assertNull( - "clusterstate JSON serialization does not match", - differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap) - ); - } catch (AssertionError error) { - logger.error( - "Cluster state from master:\n{}\nLocal cluster state:\n{}", - masterClusterState.toString(), - localClusterState.toString() - ); - throw error; - } - } - } + doEnsureClusterStateConsistency(new NamedWriteableRegistry(entries)); } } From b7b1872dfac518a36fad97cf824b2348ed815ff7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 12 Aug 2024 13:27:05 +0200 Subject: [PATCH 37/37] Refactor request marking for serverless and operator modes (#110370) The internal REST parameter `path_restricted` was introduced to signal to APIs that a request should be subject to partial restrictions in Serverless mode. The semantics of that parameter have since changed, subtly: in the initial iteration, only requests that corresponded to APIs with partial restrictions were marked. Since then, we have shifted to mark _all_ requests to public APIs by non-operator users, and relying on downstream API handlers to apply partial restrictions (if they had any). This PR replaces the parameter with two new parameters: `serverless_request` and `operator_request`. These parameters are set independently of each other: * `serverless_request` is set for all requests made in serverless mode (regardless of whether the APIs are public or internal) * `operator_request` is set for equests made by operators, both in serverless and in stateful mode This gives REST handlers and other downstream code more flexibility in toggling behavior and decouples the concept of serverless mode from operator mode, which are technically two orthogonal things. Relates: ES-8753 --- .../cluster/metadata/DataStreamLifecycle.java | 2 +- .../elasticsearch/rest/BaseRestHandler.java | 9 ++- .../elasticsearch/rest/RestController.java | 8 +++ .../org/elasticsearch/rest/RestRequest.java | 64 +++++++++++++++++-- .../org/elasticsearch/rest/RestUtils.java | 8 ++- .../elasticsearch/rest/ServerlessScope.java | 5 -- .../GetDataStreamLifecycleActionTests.java | 4 +- .../metadata/DataStreamLifecycleTests.java | 10 +-- .../elasticsearch/rest/RestRequestTests.java | 35 +++++++--- .../elasticsearch/rest/RestUtilsTests.java | 18 +++--- .../operator/OperatorOnlyRegistry.java | 20 ++---- .../security/operator/OperatorPrivileges.java | 3 + .../RestGetBuiltinPrivilegesAction.java | 9 ++- .../rest/action/role/RestGetRolesAction.java | 14 +--- .../DefaultOperatorPrivilegesTests.java | 9 ++- 15 files changed, 142 insertions(+), 76 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index de9d615022975..f8c5bebca0a79 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -370,7 +370,7 @@ public static DataStreamLifecycle fromXContent(XContentParser parser) throws IOE * Adds a retention param to signal that this serialisation should include the effective retention metadata */ public static ToXContent.Params maybeAddEffectiveRetentionParams(ToXContent.Params params) { - boolean shouldAddEffectiveRetention = Objects.equals(params.param(RestRequest.PATH_RESTRICTED), "serverless"); + boolean shouldAddEffectiveRetention = params.paramAsBoolean(RestRequest.SERVERLESS_REQUEST, false); return new DelegatingMapParams( Map.of(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, Boolean.toString(shouldAddEffectiveRetention)), params diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index a17bc885f6b65..6a45d1e5dc43e 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -83,7 +83,14 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // check if the query has any parameters that are not in the supported set (if declared) Set supported = allSupportedParameters(); if (supported != null) { - var allSupported = Sets.union(RestResponse.RESPONSE_PARAMS, ALWAYS_SUPPORTED, supported); + var allSupported = Sets.union( + RestResponse.RESPONSE_PARAMS, + ALWAYS_SUPPORTED, + // these internal parameters cannot be set by end-users, but are used by Elasticsearch internally. + // they must be accepted by all handlers + RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, + supported + ); if (allSupported.containsAll(request.params().keySet()) == false) { Set unsupported = Sets.difference(request.params().keySet(), allSupported); throw new IllegalArgumentException(unrecognized(request, unsupported, allSupported, "parameter")); diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 8592888d2dd03..8e9cbd686110b 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -480,6 +480,14 @@ private void dispatchRequest( } else { threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.TRUE.toString()); } + + if (apiProtections.isEnabled()) { + // API protections are only enabled in serverless; therefore we can use this as an indicator to mark the + // request as a serverless mode request here, so downstream handlers can use the marker + request.markAsServerlessRequest(); + logger.trace("Marked request for uri [{}] as serverless request", request.uri()); + } + final var finalChannel = responseChannel; this.interceptor.intercept(request, responseChannel, handler.getConcreteRestHandler(), new ActionListener<>() { @Override diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index 66ba0c743813e..96f2c2d10dc96 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -48,7 +48,31 @@ public class RestRequest implements ToXContent.Params, Traceable { - public static final String PATH_RESTRICTED = "pathRestricted"; + /** + * Internal marker request parameter to indicate that a request was made in serverless mode. Use this parameter, together with + * {@link #OPERATOR_REQUEST} if you need to toggle behavior for serverless, for example to enforce partial API restrictions + * (prevent request fields, omit response fields) for an API. + * Requests not made in serverless mode, will *not* have this parameter set. + * Given a request instance, you can use {@link #isServerlessRequest()} to determine if the parameter is set or not. + * This is also available from {@code ToXContent.Params}. For example: + * {@code params.paramAsBoolean(RestRequest.SERVERLESS_REQUEST, false)} + */ + public static final String SERVERLESS_REQUEST = "serverlessRequest"; + /** + * Internal marker request parameter to indicate that a request was made by an operator user. + * Requests made by regular users (users without operator privileges), will *not* have this parameter set. + * Given a request instance, you can use {@link #isOperatorRequest()} to determine if the parameter is set or not. + * This is also available from {@code ToXContent.Params}. For example: + * {@code params.paramAsBoolean(RestRequest.OPERATOR_REQUEST, false)} + */ + public static final String OPERATOR_REQUEST = "operatorRequest"; + + /** + * Internal request parameters used as markers to indicate various operations modes such as serverless mode, or operator mode. + * These can never be set directly by end-users. Instead, they are set internally by Elasticsearch and must be supported by all + * request handlers. + */ + public static final Set INTERNAL_MARKER_REQUEST_PARAMETERS = Set.of(SERVERLESS_REQUEST, OPERATOR_REQUEST); // tchar pattern as defined by RFC7230 section 3.2.6 private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-Z0-9!#$%&'*+\\-.\\^_`|~]+"); @@ -616,13 +640,41 @@ public boolean hasExplicitRestApiVersion() { return restApiVersion.isPresent(); } - public void markPathRestricted(String restriction) { - if (params.containsKey(PATH_RESTRICTED)) { - throw new IllegalArgumentException("The parameter [" + PATH_RESTRICTED + "] is already defined."); + /** + * See {@link #SERVERLESS_REQUEST} + */ + public void markAsServerlessRequest() { + setParamTrueOnceAndConsume(SERVERLESS_REQUEST); + } + + /** + * See {@link #SERVERLESS_REQUEST} + */ + public boolean isServerlessRequest() { + return paramAsBoolean(SERVERLESS_REQUEST, false); + } + + /** + * See {@link #OPERATOR_REQUEST} + */ + public void markAsOperatorRequest() { + setParamTrueOnceAndConsume(OPERATOR_REQUEST); + } + + /** + * See {@link #OPERATOR_REQUEST} + */ + public boolean isOperatorRequest() { + return paramAsBoolean(OPERATOR_REQUEST, false); + } + + private void setParamTrueOnceAndConsume(String param) { + if (params.containsKey(param)) { + throw new IllegalArgumentException("The parameter [" + param + "] is already defined."); } - params.put(PATH_RESTRICTED, restriction); + params.put(param, "true"); // this parameter is intended be consumed via ToXContent.Params.param(..), not this.params(..) so don't require it is consumed here - consumedParams.add(PATH_RESTRICTED); + consumedParams.add(param); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java index 0e7200fa83b1c..681f4c33eb77c 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java +++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java @@ -23,7 +23,7 @@ import java.util.regex.Pattern; import static org.elasticsearch.action.support.master.AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS; public class RestUtils { @@ -85,8 +85,10 @@ private static String decodeQueryStringParam(final String s) { } private static void addParam(Map params, String name, String value) { - if (PATH_RESTRICTED.equalsIgnoreCase(name)) { - throw new IllegalArgumentException("parameter [" + PATH_RESTRICTED + "] is reserved and may not set"); + for (var reservedParameter : INTERNAL_MARKER_REQUEST_PARAMETERS) { + if (reservedParameter.equalsIgnoreCase(name)) { + throw new IllegalArgumentException("parameter [" + name + "] is reserved and may not be set"); + } } params.put(name, value); } diff --git a/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java b/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java index 34aa04c5e484b..8a078db7dc012 100644 --- a/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java +++ b/server/src/main/java/org/elasticsearch/rest/ServerlessScope.java @@ -22,9 +22,4 @@ @Target(ElementType.TYPE) public @interface ServerlessScope { Scope value(); - - /** - * A value used when restricting a response of a serverless endpoints. - */ - String SERVERLESS_RESTRICTION = "serverless"; } diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java index c769e504ef15b..5c858acc2d73e 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.util.Map; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST; import static org.hamcrest.Matchers.equalTo; public class GetDataStreamLifecycleActionTests extends ESTestCase { @@ -75,7 +75,7 @@ private Map getXContentMap( TimeValue globalMaxRetention ) throws IOException { try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { - ToXContent.Params params = new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless")); + ToXContent.Params params = new ToXContent.MapParams(Map.of(SERVERLESS_REQUEST, "true")); RolloverConfiguration rolloverConfiguration = null; DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 50ab76ed794d8..7bd51aa35cba3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -39,7 +39,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DATA_STREAM_CONFIGURATION; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DEFAULT_GLOBAL_RETENTION; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.MAX_GLOBAL_RETENTION; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -354,13 +354,7 @@ public void testEffectiveRetentionParams() { } { ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams( - new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "not-serverless")) - ); - assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(false)); - } - { - ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams( - new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless")) + new ToXContent.MapParams(Map.of(SERVERLESS_REQUEST, "true")) ); assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(true)); } diff --git a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java index bb06dbe5d09aa..ae88215f951de 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestRequestTests.java @@ -31,7 +31,8 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.OPERATOR_REQUEST; +import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -249,16 +250,30 @@ public void testRequiredContent() { assertEquals("unknown content type", e.getMessage()); } - public void testMarkPathRestricted() { + public void testIsServerlessRequest() { RestRequest request1 = contentRestRequest("content", new HashMap<>()); - request1.markPathRestricted("foo"); - assertEquals(request1.param(PATH_RESTRICTED), "foo"); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> request1.markPathRestricted("foo")); - assertThat(exception.getMessage(), is("The parameter [" + PATH_RESTRICTED + "] is already defined.")); - - RestRequest request2 = contentRestRequest("content", Map.of(PATH_RESTRICTED, "foo")); - exception = expectThrows(IllegalArgumentException.class, () -> request2.markPathRestricted("bar")); - assertThat(exception.getMessage(), is("The parameter [" + PATH_RESTRICTED + "] is already defined.")); + request1.markAsServerlessRequest(); + assertEquals(request1.param(SERVERLESS_REQUEST), "true"); + assertTrue(request1.isServerlessRequest()); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, request1::markAsServerlessRequest); + assertThat(exception.getMessage(), is("The parameter [" + SERVERLESS_REQUEST + "] is already defined.")); + + RestRequest request2 = contentRestRequest("content", Map.of(SERVERLESS_REQUEST, "true")); + exception = expectThrows(IllegalArgumentException.class, request2::markAsServerlessRequest); + assertThat(exception.getMessage(), is("The parameter [" + SERVERLESS_REQUEST + "] is already defined.")); + } + + public void testIsOperatorRequest() { + RestRequest request1 = contentRestRequest("content", new HashMap<>()); + request1.markAsOperatorRequest(); + assertEquals(request1.param(OPERATOR_REQUEST), "true"); + assertTrue(request1.isOperatorRequest()); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, request1::markAsOperatorRequest); + assertThat(exception.getMessage(), is("The parameter [" + OPERATOR_REQUEST + "] is already defined.")); + + RestRequest request2 = contentRestRequest("content", Map.of(OPERATOR_REQUEST, "true")); + exception = expectThrows(IllegalArgumentException.class, request2::markAsOperatorRequest); + assertThat(exception.getMessage(), is("The parameter [" + OPERATOR_REQUEST + "] is already defined.")); } public static RestRequest contentRestRequest(String content, Map params) { diff --git a/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java b/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java index 3226ca2bf51d2..24d40fd1b95fd 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java @@ -18,7 +18,7 @@ import java.util.Map; import java.util.regex.Pattern; -import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.elasticsearch.rest.RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -160,13 +160,15 @@ public void testCrazyURL() { } public void testReservedParameters() { - Map params = new HashMap<>(); - String uri = "something?" + PATH_RESTRICTED + "=value"; - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params) - ); - assertEquals(exception.getMessage(), "parameter [" + PATH_RESTRICTED + "] is reserved and may not set"); + for (var reservedParam : INTERNAL_MARKER_REQUEST_PARAMETERS) { + Map params = new HashMap<>(); + String uri = "something?" + reservedParam + "=value"; + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params) + ); + assertEquals(exception.getMessage(), "parameter [" + reservedParam + "] is reserved and may not be set"); + } } private void assertCorsSettingRegexIsNull(String settingsValue) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java index f0889f1c48c75..ef3070f0bd787 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.security.operator; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.transport.TransportRequest; @@ -23,21 +22,10 @@ public interface OperatorOnlyRegistry { OperatorPrivilegesViolation check(String action, TransportRequest request); /** - * Checks to see if a given {@link RestHandler} is subject to operator-only restrictions for the REST API. - * - * Any REST API may be fully or partially restricted. - * A fully restricted REST API mandates that the implementation of this method throw an - * {@link org.elasticsearch.ElasticsearchStatusException} with an appropriate status code and error message. - * - * A partially restricted REST API mandates that the {@link RestRequest} is marked as restricted so that the downstream handler can - * behave appropriately. - * For example, to restrict the REST response the implementation - * should call {@link RestRequest#markPathRestricted(String)} so that the downstream handler can properly restrict the response - * before returning to the client. Note - a partial restriction should not throw an exception. - * - * @param restHandler The {@link RestHandler} to check for any restrictions - * @param restRequest The {@link RestRequest} to check for any restrictions and mark any partially restricted REST API's - * @throws ElasticsearchStatusException if the request should be denied in its entirety (fully restricted) + * This method is only called if the user is not an operator. + * Implementations should fail the request if the {@link RestRequest} is not allowed to proceed by throwing an + * {@link org.elasticsearch.ElasticsearchException}. If the request should be handled by the associated {@link RestHandler}, + * then this implementations should do nothing. */ void checkRest(RestHandler restHandler, RestRequest restRequest) throws ElasticsearchException; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java index 79c529eb3d7b1..9ef41fad12401 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java @@ -182,6 +182,9 @@ public boolean checkRest(RestHandler restHandler, RestRequest restRequest, RestC ); throw e; } + } else { + restRequest.markAsOperatorRequest(); + logger.trace("Marked request for uri [{}] as operator request", restRequest.uri()); } return true; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java index 5f0657079e694..e0ef46dc73a18 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java @@ -85,9 +85,9 @@ public RestResponse buildResponse(GetBuiltinPrivilegesResponse response, XConten @Override protected Exception innerCheckFeatureAvailable(RestRequest request) { - final boolean restrictPath = request.hasParam(RestRequest.PATH_RESTRICTED); - assert false == restrictPath || DiscoveryNode.isStateless(settings); - if (false == restrictPath) { + final boolean shouldRestrictForServerless = shouldRestrictForServerless(request); + assert false == shouldRestrictForServerless || DiscoveryNode.isStateless(settings); + if (false == shouldRestrictForServerless) { return super.innerCheckFeatureAvailable(request); } // This is a temporary hack: we are re-using the native roles setting as an overall feature flag for custom roles. @@ -106,4 +106,7 @@ protected Exception innerCheckFeatureAvailable(RestRequest request) { } } + private boolean shouldRestrictForServerless(RestRequest request) { + return request.isServerlessRequest() && false == request.isOperatorRequest(); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java index 232d74d16725d..dc9ecbbc63a8d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.security.rest.action.role; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.RestApiVersion; @@ -54,9 +53,9 @@ public String getName() { @Override public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { final String[] roles = request.paramAsStringArray("name", Strings.EMPTY_ARRAY); - final boolean restrictRequest = isPathRestricted(request); + final boolean restrictToNativeRolesOnly = request.isServerlessRequest() && false == request.isOperatorRequest(); return channel -> new GetRolesRequestBuilder(client).names(roles) - .nativeOnly(restrictRequest) + .nativeOnly(restrictToNativeRolesOnly) .execute(new RestBuilderListener<>(channel) { @Override public RestResponse buildResponse(GetRolesResponse response, XContentBuilder builder) throws Exception { @@ -84,17 +83,10 @@ protected Exception innerCheckFeatureAvailable(RestRequest request) { // Note: For non-restricted requests this action handles both reserved roles and native // roles, and should still be available even if native role management is disabled. // For restricted requests it should only be available if native role management is enabled - final boolean restrictPath = isPathRestricted(request); - if (false == restrictPath) { + if (false == request.isServerlessRequest() || request.isOperatorRequest()) { return null; } else { return super.innerCheckFeatureAvailable(request); } } - - private boolean isPathRestricted(RestRequest request) { - final boolean restrictRequest = request.hasParam(RestRequest.PATH_RESTRICTED); - assert false == restrictRequest || DiscoveryNode.isStateless(settings); - return restrictRequest; - } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java index aa95ea097413c..a5dabe8c2dd82 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/DefaultOperatorPrivilegesTests.java @@ -89,7 +89,7 @@ public void testWillNotCheckWhenLicenseDoesNotSupport() { verifyNoMoreInteractions(operatorOnlyRegistry); } - public void testMarkOperatorUser() throws IllegalAccessException { + public void testMarkOperatorUser() { final Settings settings = Settings.builder().put("xpack.security.operator_privileges.enabled", true).build(); when(xPackLicenseState.isAllowed(Security.OPERATOR_PRIVILEGES_FEATURE)).thenReturn(true); final User operatorUser = new User("operator_user"); @@ -204,7 +204,7 @@ public void testCheckWillPassForInternalUsersBecauseTheyHaveOperatorPrivileges() verify(operatorOnlyRegistry, never()).check(anyString(), any()); } - public void testMaybeInterceptRequest() throws IllegalAccessException { + public void testMaybeInterceptRequest() { final boolean licensed = randomBoolean(); when(xPackLicenseState.isAllowed(Security.OPERATOR_PRIVILEGES_FEATURE)).thenReturn(licensed); @@ -279,11 +279,16 @@ public void testCheckRest() { ); assertThat(ex, instanceOf(ElasticsearchSecurityException.class)); assertThat(ex, throwableWithMessage("violation!")); + verify(restRequest, never()).markAsOperatorRequest(); Mockito.clearInvocations(operatorOnlyRegistry); + Mockito.clearInvocations(restRequest); // is an operator threadContext.putHeader(AuthenticationField.PRIVILEGE_CATEGORY_KEY, AuthenticationField.PRIVILEGE_CATEGORY_VALUE_OPERATOR); verifyNoInteractions(operatorOnlyRegistry); assertTrue(operatorPrivilegesService.checkRest(restHandler, restRequest, restChannel, threadContext)); + verify(restRequest, times(1)).markAsOperatorRequest(); + Mockito.clearInvocations(operatorOnlyRegistry); + Mockito.clearInvocations(restRequest); } }