diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh index f2dc40ca1927f..bbfa81f51b286 100755 --- a/.buildkite/scripts/dra-workflow.sh +++ b/.buildkite/scripts/dra-workflow.sh @@ -75,6 +75,7 @@ find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \; echo --- Running release-manager +set +e # Artifacts should be generated docker run --rm \ --name release-manager \ @@ -91,4 +92,16 @@ docker run --rm \ --version "$ES_VERSION" \ --artifact-set main \ --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \ - --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" + --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \ +2>&1 | tee release-manager.log +EXIT_CODE=$? +set -e + +# This failure is just generating a ton of noise right now, so let's just ignore it +# This should be removed once this issue has been fixed +if grep "elasticsearch-ubi-9.0.0-SNAPSHOT-docker-image.tar.gz" release-manager.log; then + echo "Ignoring error about missing ubi artifact" + exit 0 +fi + +exit "$EXIT_CODE" diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java index 9d06e632ec928..89bab313a0069 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java @@ -16,6 +16,8 @@ import org.gradle.api.file.RegularFileProperty; import org.gradle.api.model.ObjectFactory; import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import java.io.FileReader; @@ -37,6 +39,7 @@ public PomValidationTask(ObjectFactory objects) { } @InputFile + @PathSensitive(PathSensitivity.RELATIVE) public RegularFileProperty getPomFile() { return pomFile; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index d7bf839817e12..5992a40275b46 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -115,9 +115,9 @@ private static TaskProvider createRunBwcGradleTask( if (OS.current() == OS.WINDOWS) { loggedExec.getExecutable().set("cmd"); - loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString()); + loggedExec.args("/C", "call", "gradlew"); } else { - loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString()); + loggedExec.getExecutable().set("./gradlew"); } if (useUniqueUserHome) { @@ -177,7 +177,7 @@ private static String readFromFile(File file) { } } - public static abstract class JavaHomeValueSource implements ValueSource { + public abstract static class JavaHomeValueSource implements ValueSource { private String minimumCompilerVersionPath(Version bwcVersion) { return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION)) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index bf901fef90450..71e968557cefe 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -22,7 +22,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8", "-wolfi", "apk" ), diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 80fd6db59cf9f..c17127f9bbfcf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -23,6 +23,7 @@ import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskProvider; import org.gradle.jvm.toolchain.JavaToolchainService; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -322,7 +323,7 @@ static void createBuildBwcTask( File expectedOutputFile = useNativeExpanded ? new File(projectArtifact.expandedDistDir, "elasticsearch-" + bwcVersion.get() + "-SNAPSHOT") : projectArtifact.distFile; - c.getInputs().file(new File(project.getBuildDir(), "refspec")); + c.getInputs().file(new File(project.getBuildDir(), "refspec")).withPathSensitivity(PathSensitivity.RELATIVE); if (useNativeExpanded) { c.getOutputs().dir(expectedOutputFile); } else { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java index 81ff081ffa82b..dbbe35905d208 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java @@ -19,6 +19,7 @@ import org.gradle.api.plugins.quality.Checkstyle; import org.gradle.api.plugins.quality.CheckstyleExtension; import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.TaskProvider; @@ -42,18 +43,23 @@ public TaskProvider createTask(Project project) { File checkstyleSuppressions = new File(checkstyleDir, "checkstyle_suppressions.xml"); File checkstyleConf = new File(checkstyleDir, "checkstyle.xml"); TaskProvider copyCheckstyleConf = project.getTasks().register("copyCheckstyleConf"); - // configure inputs and outputs so up to date works properly copyCheckstyleConf.configure(t -> t.getOutputs().files(checkstyleSuppressions, checkstyleConf)); if ("jar".equals(checkstyleConfUrl.getProtocol())) { try { JarURLConnection jarURLConnection = (JarURLConnection) checkstyleConfUrl.openConnection(); - copyCheckstyleConf.configure(t -> t.getInputs().file(jarURLConnection.getJarFileURL())); + copyCheckstyleConf.configure( + t -> t.getInputs().file(jarURLConnection.getJarFileURL()).withPathSensitivity(PathSensitivity.RELATIVE) + ); } catch (IOException e) { throw new UncheckedIOException(e); } } else if ("file".equals(checkstyleConfUrl.getProtocol())) { - copyCheckstyleConf.configure(t -> t.getInputs().files(checkstyleConfUrl.getFile(), checkstyleSuppressionsUrl.getFile())); + copyCheckstyleConf.configure( + t -> t.getInputs() + .files(checkstyleConfUrl.getFile(), checkstyleSuppressionsUrl.getFile()) + .withPathSensitivity(PathSensitivity.RELATIVE) + ); } // Explicitly using an Action interface as java lambdas diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java index a198034c3c09b..479b6f431b867 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java @@ -19,6 +19,8 @@ import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.SkipWhenEmpty; import org.gradle.api.tasks.StopExecutionException; import org.gradle.api.tasks.TaskAction; @@ -79,6 +81,7 @@ private static boolean isExecutableFile(File file) { @InputFiles @IgnoreEmptyDirectories @SkipWhenEmpty + @PathSensitive(PathSensitivity.RELATIVE) public FileCollection getFiles() { return getSources().get() .stream() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java index 02309bb9c1811..6890cfb652952 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java @@ -24,6 +24,8 @@ import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.SkipWhenEmpty; import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.util.PatternFilterable; @@ -106,6 +108,7 @@ public Map getSubstitutions() { @SkipWhenEmpty @IgnoreEmptyDirectories @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) public FileTree getInputDir() { FileTree coreFileTree = null; FileTree xpackFileTree = null; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index 505e9a5b114d1..28018b4c50abe 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -20,6 +20,7 @@ import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Optional; @@ -53,6 +54,7 @@ * Exec task implementation. */ @SuppressWarnings("unchecked") +@CacheableTask public abstract class LoggedExec extends DefaultTask implements FileSystemOperationsAware { private static final Logger LOGGER = Logging.getLogger(LoggedExec.class); @@ -87,6 +89,14 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat abstract public Property getCaptureOutput(); @Input + public Provider getWorkingDirPath() { + return getWorkingDir().map(file -> { + String relativeWorkingDir = projectLayout.getProjectDirectory().getAsFile().toPath().relativize(file.toPath()).toString(); + return relativeWorkingDir; + }); + } + + @Internal abstract public Property getWorkingDir(); @Internal @@ -117,9 +127,10 @@ public LoggedExec( * can be reused across different build invocations. * */ private void setupDefaultEnvironment(ProviderFactory providerFactory) { - getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE")); getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("GRADLE_BUILD_CACHE")); - getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT")); + + getNonTrackedEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE")); + getNonTrackedEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT")); Provider javaToolchainHome = providerFactory.environmentVariable("JAVA_TOOLCHAIN_HOME"); if (javaToolchainHome.isPresent()) { getEnvironment().put("JAVA_TOOLCHAIN_HOME", javaToolchainHome); diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java index e144122f97770..6cf01814a45ef 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java @@ -19,10 +19,13 @@ import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.tree.ClassNode; @@ -39,6 +42,7 @@ import javax.inject.Inject; +@CacheableTask public abstract class GeneratePluginPropertiesTask extends DefaultTask { public static final String PROPERTIES_FILENAME = "plugin-descriptor.properties"; @@ -82,6 +86,7 @@ public GeneratePluginPropertiesTask(ProjectLayout projectLayout) { public abstract Property getIsLicensed(); @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) public abstract ConfigurableFileCollection getModuleInfoFile(); @OutputFile diff --git a/docs/changelog/115020.yaml b/docs/changelog/115020.yaml new file mode 100644 index 0000000000000..2b0aefafea507 --- /dev/null +++ b/docs/changelog/115020.yaml @@ -0,0 +1,5 @@ +pr: 115020 +summary: Adding endpoint creation validation for all task types to remaining services +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/117246.yaml b/docs/changelog/117246.yaml new file mode 100644 index 0000000000000..29c4464855967 --- /dev/null +++ b/docs/changelog/117246.yaml @@ -0,0 +1,5 @@ +pr: 117246 +summary: LOOKUP JOIN using field-caps for field mapping +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117312.yaml b/docs/changelog/117312.yaml new file mode 100644 index 0000000000000..302b91388ef2b --- /dev/null +++ b/docs/changelog/117312.yaml @@ -0,0 +1,5 @@ +pr: 117312 +summary: Add missing `async_search` query parameters to rest-api-spec +area: Search +type: bug +issues: [] diff --git a/docs/changelog/117404.yaml b/docs/changelog/117404.yaml new file mode 100644 index 0000000000000..0bab171956ca9 --- /dev/null +++ b/docs/changelog/117404.yaml @@ -0,0 +1,5 @@ +pr: 117404 +summary: Correct bit * byte and bit * float script comparisons +area: Vector Search +type: bug +issues: [] diff --git a/docs/reference/data-streams/tsds-reindex.asciidoc b/docs/reference/data-streams/tsds-reindex.asciidoc index 9d6594db4e779..f4d00f33c179c 100644 --- a/docs/reference/data-streams/tsds-reindex.asciidoc +++ b/docs/reference/data-streams/tsds-reindex.asciidoc @@ -202,7 +202,7 @@ POST /_component_template/destination_template POST /_index_template/2 { "index_patterns": [ - "k8s*" + "k9s*" ], "composed_of": [ "destination_template" diff --git a/docs/reference/esql/esql-kibana.asciidoc b/docs/reference/esql/esql-kibana.asciidoc index 85969e19957af..87dd4d87fa8e3 100644 --- a/docs/reference/esql/esql-kibana.asciidoc +++ b/docs/reference/esql/esql-kibana.asciidoc @@ -106,12 +106,27 @@ detailed warning, expand the query bar, and click *warnings*. ==== Query history You can reuse your recent {esql} queries in the query bar. -In the query bar click *Show recent queries*. +In the query bar, click *Show recent queries*. You can then scroll through your recent queries: image::images/esql/esql-discover-query-history.png[align="center",size="50%"] +[discrete] +[[esql-kibana-starred-queries]] +==== Starred queries + +From the query history, you can mark some queries as favorite to find and access them faster later. + +In the query bar, click *Show recent queries*. + +From the **Recent** tab, you can star any queries you want. + +In the **Starred** tab, find all the queries you have previously starred. + +image::images/esql/esql-discover-query-starred.png[align="center",size="50%"] + + [discrete] [[esql-kibana-results-table]] === The results table diff --git a/docs/reference/esql/functions/description/kql.asciidoc b/docs/reference/esql/functions/description/kql.asciidoc new file mode 100644 index 0000000000000..e1fe411e6689c --- /dev/null +++ b/docs/reference/esql/functions/description/kql.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a KQL query. Returns true if the provided KQL query string matches the row. diff --git a/docs/reference/esql/functions/examples/kql.asciidoc b/docs/reference/esql/functions/examples/kql.asciidoc new file mode 100644 index 0000000000000..1f8518aeec394 --- /dev/null +++ b/docs/reference/esql/functions/examples/kql.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/kql-function.csv-spec[tag=kql-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/kql-function.csv-spec[tag=kql-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/kql.json b/docs/reference/esql/functions/kibana/definition/kql.json new file mode 100644 index 0000000000000..6960681fbbf0d --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/kql.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "kql", + "description" : "Performs a KQL query. Returns true if the provided KQL query string matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in KQL query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in KQL query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE KQL(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/kql.md b/docs/reference/esql/functions/kibana/docs/kql.md new file mode 100644 index 0000000000000..0ba419c1cd032 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/kql.md @@ -0,0 +1,14 @@ + + +### KQL +Performs a KQL query. Returns true if the provided KQL query string matches the row. + +``` +FROM books +| WHERE KQL("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +``` diff --git a/docs/reference/esql/functions/layout/kql.asciidoc b/docs/reference/esql/functions/layout/kql.asciidoc new file mode 100644 index 0000000000000..8cf2687b240c1 --- /dev/null +++ b/docs/reference/esql/functions/layout/kql.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-kql]] +=== `KQL` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/kql.svg[Embedded,opts=inline] + +include::../parameters/kql.asciidoc[] +include::../description/kql.asciidoc[] +include::../types/kql.asciidoc[] +include::../examples/kql.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/kql.asciidoc b/docs/reference/esql/functions/parameters/kql.asciidoc new file mode 100644 index 0000000000000..6fb69323ff73c --- /dev/null +++ b/docs/reference/esql/functions/parameters/kql.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`query`:: +Query string in KQL query string format. diff --git a/docs/reference/esql/functions/signature/kql.svg b/docs/reference/esql/functions/signature/kql.svg new file mode 100644 index 0000000000000..3f550f27ccdff --- /dev/null +++ b/docs/reference/esql/functions/signature/kql.svg @@ -0,0 +1 @@ +KQL(query) \ No newline at end of file diff --git a/docs/reference/esql/functions/spatial-functions.asciidoc b/docs/reference/esql/functions/spatial-functions.asciidoc index 79acc2028d983..eee44d337b4c6 100644 --- a/docs/reference/esql/functions/spatial-functions.asciidoc +++ b/docs/reference/esql/functions/spatial-functions.asciidoc @@ -8,19 +8,19 @@ {esql} supports these spatial functions: // tag::spatial_list[] -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> // end::spatial_list[] +include::layout/st_distance.asciidoc[] include::layout/st_intersects.asciidoc[] include::layout/st_disjoint.asciidoc[] include::layout/st_contains.asciidoc[] include::layout/st_within.asciidoc[] include::layout/st_x.asciidoc[] include::layout/st_y.asciidoc[] -include::layout/st_distance.asciidoc[] diff --git a/docs/reference/esql/functions/types/kql.asciidoc b/docs/reference/esql/functions/types/kql.asciidoc new file mode 100644 index 0000000000000..866a39e925665 --- /dev/null +++ b/docs/reference/esql/functions/types/kql.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +query | result +keyword | boolean +text | boolean +|=== diff --git a/docs/reference/geospatial-analysis.asciidoc b/docs/reference/geospatial-analysis.asciidoc index 6760040e14bc7..678e0ee17aec2 100644 --- a/docs/reference/geospatial-analysis.asciidoc +++ b/docs/reference/geospatial-analysis.asciidoc @@ -38,11 +38,11 @@ Data is often messy and incomplete. <> lets you clean, <> has support for <> functions, enabling efficient index searching for documents that intersect with, are within, are contained by, or are disjoint from a query geometry. In addition, the `ST_DISTANCE` function calculates the distance between two points. -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> +* <> +* <> +* <> +* <> +* <> [discrete] [[geospatial-aggregate]] diff --git a/docs/reference/images/esql/esql-discover-query-history.png b/docs/reference/images/esql/esql-discover-query-history.png index ff1d2ffa8b280..eb064684af700 100644 Binary files a/docs/reference/images/esql/esql-discover-query-history.png and b/docs/reference/images/esql/esql-discover-query-history.png differ diff --git a/docs/reference/images/esql/esql-discover-query-starred.png b/docs/reference/images/esql/esql-discover-query-starred.png new file mode 100644 index 0000000000000..525aa9acbea28 Binary files /dev/null and b/docs/reference/images/esql/esql-discover-query-starred.png differ diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 4c16f260c13e7..e6e11d6dd539f 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -338,12 +338,12 @@ by 32x at the cost of accuracy. See <> to smaller secti The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the <>. This field type and the <> type make it simpler to perform semantic search on your data. +If you don't specify an inference endpoint, the <> is used by default. Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it. The {infer} endpoint automatically determines the embedding generation, indexing, and query to use. +If you use the ELSER service, you can set up `semantic_text` with the following API request: + [source,console] ------------------------------------------------------------ PUT my-index-000001 +{ + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text" + } + } + } +} +------------------------------------------------------------ + +If you use a service other than ELSER, you must create an {infer} endpoint using the <> and reference it when setting up `semantic_text` as the following example demonstrates: + +[source,console] +------------------------------------------------------------ +PUT my-index-000002 { "mappings": { "properties": { "inference_field": { "type": "semantic_text", - "inference_id": "my-elser-endpoint" + "inference_id": "my-openai-endpoint" <1> } } } } ------------------------------------------------------------ // TEST[skip:Requires inference endpoint] +<1> The `inference_id` of the {infer} endpoint to use to generate embeddings. The recommended way to use semantic_text is by having dedicated {infer} endpoints for ingestion and search. @@ -40,7 +60,7 @@ After creating dedicated {infer} endpoints for both, you can reference them usin [source,console] ------------------------------------------------------------ -PUT my-index-000002 +PUT my-index-000003 { "mappings": { "properties": { diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index 60692c19c184a..ba9c81db21384 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -21,45 +21,9 @@ This tutorial uses the <> for demonstra [[semantic-text-requirements]] ==== Requirements -To use the `semantic_text` field type, you must have an {infer} endpoint deployed in -your cluster using the <>. +This tutorial uses the <> for demonstration, which is created automatically as needed. +To use the `semantic_text` field type with an {infer} service other than ELSER, you must create an inference endpoint using the <>. -[discrete] -[[semantic-text-infer-endpoint]] -==== Create the {infer} endpoint - -Create an inference endpoint by using the <>: - -[source,console] ------------------------------------------------------------- -PUT _inference/sparse_embedding/my-elser-endpoint <1> -{ - "service": "elser", <2> - "service_settings": { - "adaptive_allocations": { <3> - "enabled": true, - "min_number_of_allocations": 3, - "max_number_of_allocations": 10 - }, - "num_threads": 1 - } -} ------------------------------------------------------------- -// TEST[skip:TBD] -<1> The task type is `sparse_embedding` in the path as the `elser` service will -be used and ELSER creates sparse vectors. The `inference_id` is -`my-elser-endpoint`. -<2> The `elser` service is used in this example. -<3> This setting enables and configures {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations]. -Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. - -[NOTE] -==== -You might see a 502 bad gateway error in the response when using the {kib} Console. -This error usually just reflects a timeout, while the model downloads in the background. -You can check the download progress in the {ml-app} UI. -If using the Python client, you can set the `timeout` parameter to a higher value. -==== [discrete] [[semantic-text-index-mapping]] @@ -75,8 +39,7 @@ PUT semantic-embeddings "mappings": { "properties": { "content": { <1> - "type": "semantic_text", <2> - "inference_id": "my-elser-endpoint" <3> + "type": "semantic_text" <2> } } } @@ -85,18 +48,14 @@ PUT semantic-embeddings // TEST[skip:TBD] <1> The name of the field to contain the generated embeddings. <2> The field to contain the embeddings is a `semantic_text` field. -<3> The `inference_id` is the inference endpoint you created in the previous step. -It will be used to generate the embeddings based on the input text. -Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text. +Since no `inference_id` is provided, the <> is used by default. +To use a different {infer} service, you must create an {infer} endpoint first using the <> and then specify it in the `semantic_text` field mapping using the `inference_id` parameter. [NOTE] ==== -If you're using web crawlers or connectors to generate indices, you have to -<> for these indices to -include the `semantic_text` field. Once the mapping is updated, you'll need to run -a full web crawl or a full connector sync. This ensures that all existing -documents are reprocessed and updated with the new semantic embeddings, -enabling semantic search on the updated data. +If you're using web crawlers or connectors to generate indices, you have to <> for these indices to include the `semantic_text` field. +Once the mapping is updated, you'll need to run a full web crawl or a full connector sync. +This ensures that all existing documents are reprocessed and updated with the new semantic embeddings, enabling semantic search on the updated data. ==== @@ -288,4 +247,4 @@ query from the `semantic-embedding` index: * If you want to use `semantic_text` in hybrid search, refer to https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[this notebook] for a step-by-step guide. * For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation. -* To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page. \ No newline at end of file +* To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page. diff --git a/docs/reference/vectors/vector-functions.asciidoc b/docs/reference/vectors/vector-functions.asciidoc index 10dca8084e28a..23419e8eb12b1 100644 --- a/docs/reference/vectors/vector-functions.asciidoc +++ b/docs/reference/vectors/vector-functions.asciidoc @@ -336,6 +336,10 @@ When using `bit` vectors, not all the vector functions are available. The suppor this is the sum of the bitwise AND of the two vectors. If providing `float[]` or `byte[]`, who has `dims` number of elements, as a query vector, the `dotProduct` is the sum of the floating point values using the stored `bit` vector as a mask. +NOTE: When comparing `floats` and `bytes` with `bit` vectors, the `bit` vector is treated as a mask in big-endian order. +For example, if the `bit` vector is `10100001` (e.g. the single byte value `161`) and its compared +with array of values `[1, 2, 3, 4, 5, 6, 7, 8]` the `dotProduct` will be `1 + 3 + 8 = 16`. + Here is an example of using dot-product with bit vectors. [source,console] diff --git a/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc b/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc index 421c69619cfea..b1c776baae1de 100644 --- a/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc +++ b/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc @@ -62,20 +62,20 @@ contain the words "error" or "problem". To set up the watch: -. Define the watch trigger--a daily schedule that runs at 12:00 UTC: +. Define the watch trigger--a daily schedule that runs at 12:00 Australian Eastern Standard Time (UTC+10:00): + [source,js] -------------------------------------------------- "trigger" : { "schedule" : { + "timezone": "Australia/Brisbane", "daily" : { "at" : "12:00" } } } -------------------------------------------------- + -NOTE: In {watcher}, you specify times in UTC time. Don't forget to do the - conversion from your local time so the schedule triggers at the time - you intend. +NOTE: In {watcher}, if the timezone is omitted then schedules default to UTC. `timezone` can be specified either +as a +/-HH:mm offset from UTC or as a timezone name from the machines local IANA Time Zone Database. . Define the watch input--a search that uses a filter to constrain the results to the past day. diff --git a/docs/reference/watcher/trigger/schedule.asciidoc b/docs/reference/watcher/trigger/schedule.asciidoc index fa389409d15c4..d2bf466644e10 100644 --- a/docs/reference/watcher/trigger/schedule.asciidoc +++ b/docs/reference/watcher/trigger/schedule.asciidoc @@ -6,12 +6,42 @@ ++++ Schedule <> define when the watch execution should start based -on date and time. All times are specified in UTC time. +on date and time. All times are in UTC time unless a timezone is explicitly specified +in the schedule. {watcher} uses the system clock to determine the current time. To ensure schedules are triggered when expected, you should synchronize the clocks of all nodes in the cluster using a time service such as http://www.ntp.org/[NTP]. +NOTE: {watcher} can't correct for manual adjustments to the system clock. Be aware when making +such changes that watch execution may be affected with watches being skipped or repeated if the +adjustment covers their target execution time. This applies to changes made via NTP as well. + +When specifying a timezone for a watch, keep in mind the effect daylight savings time +transitions may have on the schedule, especially if the watch is scheduled to run +during the transition. Here's how {watcher} handles watches scheduled during discontinuities: + +==== Gap Transitions +These occur when the clock moves forward, such as when daylight savings time starts +and cause certain hours or minutes to be skipped. If your watch is scheduled to run +during a gap transition, the watch is executed at the same time as before the transition. + +Example: If a watch is scheduled to run daily at 1:30AM in the `Europe/London` time zone and +the clock moves forward one hour from 1:00AM (GMT+0) to 2:00AM (GMT+1), the watch is executed +at 2:30AM (GMT+1) which would have been 1:30AM before the transition. Subsequent executions +happen at 1:30AM (GMT+1). + +==== Overlap Transitions +These occur when the clock moves backward, such as when daylight savings time ends +and cause certain hours or minutes to be repeated. If your watch is scheduled to run +during an overlap transition, only the first occurrence of the time causes to the watch +to execute with the second being skipped. + +Example: If a watch is scheduled to run at 1:30 AM and the clock moves backward one hour +from 2:00AM to 1:00AM, the watch is executed at 1:30AM and the second occurrence after the +change is skipped. + +=== Throttling Keep in mind that the throttle period can affect when a watch is actually executed. The default throttle period is five seconds (5000 ms). If you configure a schedule that's more frequent than the throttle period, the throttle period overrides the @@ -20,6 +50,7 @@ and set the schedule to every 10 seconds, the watch is executed no more than once per minute. For more information about throttling, see <>. +=== Schedule Types {watcher} provides several types of schedule triggers: * <> diff --git a/docs/reference/watcher/trigger/schedule/cron.asciidoc b/docs/reference/watcher/trigger/schedule/cron.asciidoc index 673f350435c5f..c33bf524a8737 100644 --- a/docs/reference/watcher/trigger/schedule/cron.asciidoc +++ b/docs/reference/watcher/trigger/schedule/cron.asciidoc @@ -5,14 +5,14 @@ ++++ -Defines a <> using a <> +Defines a <> using a <> that specifiues when to execute a watch. -TIP: While cron expressions are powerful, a regularly occurring schedule -is easier to configure with the other schedule types. -If you must use a cron schedule, make sure you verify it with -<> . +TIP: While cron expressions are powerful, a regularly occurring schedule +is easier to configure with the other schedule types. +If you must use a cron schedule, make sure you verify it with +<> . ===== Configure a cron schedule with one time @@ -60,16 +60,40 @@ minute during the weekend: -------------------------------------------------- // NOTCONSOLE +[[configue_cron_time-zone]] +==== Use a different time zone for a cron schedule +By default, cron expressions are evaluated in the UTC time zone. To use a different time zone, +you can specify the `timezone` parameter in the schedule. For example, the following +`cron` schedule triggers at 6:00 AM and 6:00 PM during weekends in the `America/Los_Angeles` time zone: + + +[source,js] +-------------------------------------------------- +{ + ... + "trigger" : { + "schedule" : { + "timezone" : "America/Los_Angeles", + "cron" : [ + "0 6,18 * * * SAT-SUN", + ] + } + } + ... +} +-------------------------------------------------- +// NOTCONSOLE + [[croneval]] ===== Use croneval to validate cron expressions -{es} provides a <> command line tool -in the `$ES_HOME/bin` directory that you can use to check that your cron expressions +{es} provides a <> command line tool +in the `$ES_HOME/bin` directory that you can use to check that your cron expressions are valid and produce the expected results. -To validate a cron expression, pass it in as a parameter to `elasticsearch-croneval`: +To validate a cron expression, pass it in as a parameter to `elasticsearch-croneval`: [source,bash] -------------------------------------------------- bin/elasticsearch-croneval "0 0/1 * * * ?" --------------------------------------------------- +-------------------------------------------------- diff --git a/docs/reference/watcher/trigger/schedule/daily.asciidoc b/docs/reference/watcher/trigger/schedule/daily.asciidoc index cea2b8316e02f..d258d9c612350 100644 --- a/docs/reference/watcher/trigger/schedule/daily.asciidoc +++ b/docs/reference/watcher/trigger/schedule/daily.asciidoc @@ -97,3 +97,27 @@ or minutes as an array. For example, following `daily` schedule triggers at } -------------------------------------------------- // NOTCONSOLE + +[[specifying-time-zone-for-daily-schedule]] +===== Specifying a time zone for a daily schedule +By default, daily schedules are evaluated in the UTC time zone. To use a different time zone, +you can specify the `timezone` parameter in the schedule. For example, the following +`daily` schedule triggers at 6:00 AM and 6:00 PM in the `Pacific/Galapagos` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "Pacific/Galapagos", + "daily" : { + "at" : { + "hour" : [ 6, 18 ], + "minute" : 0 + } + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/watcher/trigger/schedule/monthly.asciidoc b/docs/reference/watcher/trigger/schedule/monthly.asciidoc index 7d13262ed2fa8..694c76aaee23a 100644 --- a/docs/reference/watcher/trigger/schedule/monthly.asciidoc +++ b/docs/reference/watcher/trigger/schedule/monthly.asciidoc @@ -74,4 +74,26 @@ schedule triggers at 12:00 AM and 12:00 PM on the 10th and 20th of each month. } } -------------------------------------------------- -// NOTCONSOLE \ No newline at end of file +// NOTCONSOLE + +==== Configuring time zones for monthly schedules +By default, monthly schedules are evaluated in the UTC time zone. To use a different +time zone, you can specify the `timezone` parameter in the schedule. For example, +the following `monthly` schedule triggers at 6:00 AM and 6:00 PM on the 15th of each month in +the `Asia/Tokyo` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "Asia/Tokyo", + "monthly" : { + "on" : [ 15 ], + "at" : [ 6:00, 18:00 ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/watcher/trigger/schedule/weekly.asciidoc b/docs/reference/watcher/trigger/schedule/weekly.asciidoc index 5b43de019ad25..53bd2f3167b21 100644 --- a/docs/reference/watcher/trigger/schedule/weekly.asciidoc +++ b/docs/reference/watcher/trigger/schedule/weekly.asciidoc @@ -79,4 +79,26 @@ Alternatively, you can specify days and times in an object that has `on` and } } -------------------------------------------------- -// NOTCONSOLE \ No newline at end of file +// NOTCONSOLE + +==== Use a different time zone for a weekly schedule +By default, weekly schedules are evaluated in the UTC time zone. To use a different time zone, +you can specify the `timezone` parameter in the schedule. For example, the following +`weekly` schedule triggers at 6:00 AM and 6:00 PM on Tuesdays and Fridays in the +`America/Buenos_Aires` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "America/Buenos_Aires", + "weekly" : { + "on" : [ "tuesday", "friday" ], + "at" : [ "6:00", "18:00" ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/watcher/trigger/schedule/yearly.asciidoc b/docs/reference/watcher/trigger/schedule/yearly.asciidoc index 8fce024bf9f4a..c33321ef5a7dc 100644 --- a/docs/reference/watcher/trigger/schedule/yearly.asciidoc +++ b/docs/reference/watcher/trigger/schedule/yearly.asciidoc @@ -88,3 +88,26 @@ January 20th, December 10th, and December 20th. } -------------------------------------------------- // NOTCONSOLE + +==== Configuring a yearly schedule with a different time zone +By default, the `yearly` schedule is evaluated in the UTC time zone. To use a +different time zone, you can specify the `timezone` parameter in the schedule. +For example, the following `yearly` schedule triggers at 3:30 PM and 8:30 PM +on June 4th in the `Antarctica/Troll` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "Antarctica/Troll", + "yearly" : { + "in" : "june", + "on" : 4, + "at" : [ 15:30, 20:30 ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java index de2cb9042610b..2f4743a47a14a 100644 --- a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java @@ -51,6 +51,8 @@ public static long ipByteBinByte(byte[] q, byte[] d) { /** * Compute the inner product of two vectors, where the query vector is a byte vector and the document vector is a bit vector. * This will return the sum of the query vector values using the document vector as a mask. + * When comparing the bits with the bytes, they are done in "big endian" order. For example, if the byte vector + * is [1, 2, 3, 4, 5, 6, 7, 8] and the bit vector is [0b10000000], the inner product will be 1.0. * @param q the query vector * @param d the document vector * @return the inner product of the two vectors @@ -63,9 +65,9 @@ public static int ipByteBit(byte[] q, byte[] d) { // now combine the two vectors, summing the byte dimensions where the bit in d is `1` for (int i = 0; i < d.length; i++) { byte mask = d[i]; - for (int j = 0; j < Byte.SIZE; j++) { + for (int j = Byte.SIZE - 1; j >= 0; j--) { if ((mask & (1 << j)) != 0) { - result += q[i * Byte.SIZE + j]; + result += q[i * Byte.SIZE + Byte.SIZE - 1 - j]; } } } @@ -75,6 +77,8 @@ public static int ipByteBit(byte[] q, byte[] d) { /** * Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a bit vector. * This will return the sum of the query vector values using the document vector as a mask. + * When comparing the bits with the floats, they are done in "big endian" order. For example, if the float vector + * is [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] and the bit vector is [0b10000000], the inner product will be 1.0. * @param q the query vector * @param d the document vector * @return the inner product of the two vectors @@ -86,9 +90,9 @@ public static float ipFloatBit(float[] q, byte[] d) { float result = 0; for (int i = 0; i < d.length; i++) { byte mask = d[i]; - for (int j = 0; j < Byte.SIZE; j++) { + for (int j = Byte.SIZE - 1; j >= 0; j--) { if ((mask & (1 << j)) != 0) { - result += q[i * Byte.SIZE + j]; + result += q[i * Byte.SIZE + Byte.SIZE - 1 - j]; } } } diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java index e9e0fd58f7638..368898b934c87 100644 --- a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java @@ -21,6 +21,22 @@ public class ESVectorUtilTests extends BaseVectorizationTests { static final ESVectorizationProvider defaultedProvider = BaseVectorizationTests.defaultProvider(); static final ESVectorizationProvider defOrPanamaProvider = BaseVectorizationTests.maybePanamaProvider(); + public void testIpByteBit() { + byte[] q = new byte[16]; + byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) }; + random().nextBytes(q); + int expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15]; + assertEquals(expected, ESVectorUtil.ipByteBit(q, d)); + } + + public void testIpFloatBit() { + float[] q = new float[16]; + byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) }; + random().nextFloat(); + float expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15]; + assertEquals(expected, ESVectorUtil.ipFloatBit(q, d), 1e-6); + } + public void testBitAndCount() { testBasicBitAndImpl(ESVectorUtil::andBitCountLong); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index a3995d7462b32..e009db7209eab 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -48,7 +48,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck( "enabled", timestampMapping(true, b -> b.startObject("@timestamp").field("type", "date").endObject()), - timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()) + timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()), + dm -> {} ); checker.registerUpdateCheck( timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()), diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml index caa7c59ab4c42..77d4b70cdfcae 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml @@ -3,7 +3,7 @@ setup: capabilities: - method: POST path: /_search - capabilities: [ multi_dense_vector_script_max_sim ] + capabilities: [ multi_dense_vector_script_max_sim_with_bugfix ] test_runner_features: capabilities reason: "Support for multi dense vector max-sim functions capability required" - skip: @@ -136,10 +136,10 @@ setup: - match: {hits.total: 2} - match: {hits.hits.0._id: "1"} - - close_to: {hits.hits.0._score: {value: 190, error: 0.01}} + - close_to: {hits.hits.0._score: {value: 220, error: 0.01}} - match: {hits.hits.1._id: "3"} - - close_to: {hits.hits.1._score: {value: 125, error: 0.01}} + - close_to: {hits.hits.1._score: {value: 147, error: 0.01}} --- "Test max-sim inv hamming scoring": - skip: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml index 2ee38f849e9d4..cdd65ca0eb296 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml @@ -108,7 +108,7 @@ setup: capabilities: - method: POST path: /_search - capabilities: [ byte_float_bit_dot_product ] + capabilities: [ byte_float_bit_dot_product_with_bugfix ] reason: Capability required to run test - do: catch: bad_request @@ -399,7 +399,7 @@ setup: capabilities: - method: POST path: /_search - capabilities: [ byte_float_bit_dot_product ] + capabilities: [ byte_float_bit_dot_product_with_bugfix ] test_runner_features: [capabilities, close_to] reason: Capability required to run test - do: @@ -419,13 +419,13 @@ setup: - match: { hits.total: 3 } - match: {hits.hits.0._id: "2"} - - close_to: {hits.hits.0._score: {value: 35.999, error: 0.01}} + - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}} - match: {hits.hits.1._id: "3"} - - close_to: {hits.hits.1._score:{value: 27.23, error: 0.01}} + - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}} - match: {hits.hits.2._id: "1"} - - close_to: {hits.hits.2._score: {value: 16.57, error: 0.01}} + - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}} - do: headers: @@ -444,20 +444,20 @@ setup: - match: { hits.total: 3 } - match: {hits.hits.0._id: "2"} - - close_to: {hits.hits.0._score: {value: 35.999, error: 0.01}} + - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}} - match: {hits.hits.1._id: "3"} - - close_to: {hits.hits.1._score:{value: 27.23, error: 0.01}} + - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}} - match: {hits.hits.2._id: "1"} - - close_to: {hits.hits.2._score: {value: 16.57, error: 0.01}} + - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}} --- "Dot product with byte": - requires: capabilities: - method: POST path: /_search - capabilities: [ byte_float_bit_dot_product ] + capabilities: [ byte_float_bit_dot_product_with_bugfix ] test_runner_features: capabilities reason: Capability required to run test - do: @@ -476,14 +476,14 @@ setup: - match: { hits.total: 3 } - - match: {hits.hits.0._id: "1"} - - match: {hits.hits.0._score: 248} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0._score: 415} - - match: {hits.hits.1._id: "2"} - - match: {hits.hits.1._score: 136} + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 168} - - match: {hits.hits.2._id: "3"} - - match: {hits.hits.2._score: 20} + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2._score: 126} - do: headers: @@ -501,11 +501,11 @@ setup: - match: { hits.total: 3 } - - match: {hits.hits.0._id: "1"} - - match: {hits.hits.0._score: 248} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0._score: 415} - - match: {hits.hits.1._id: "2"} - - match: {hits.hits.1._score: 136} + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 168} - - match: {hits.hits.2._id: "3"} - - match: {hits.hits.2._score: 20} + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2._score: 126} diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 1301d17606d63..9a7f0a5994d73 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -45,6 +45,7 @@ dependencies { testImplementation project(':test:fixtures:s3-fixture') yamlRestTestImplementation project(":test:framework") yamlRestTestImplementation project(':test:fixtures:s3-fixture') + yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture') yamlRestTestImplementation project(':test:fixtures:minio-fixture') internalClusterTestImplementation project(':test:fixtures:minio-fixture') diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java similarity index 89% rename from modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java rename to modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java index dcd29c6d26c6e..2f3e995b52468 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -28,10 +29,11 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; -public class RepositoryS3RestIT extends ESRestTestCase { +public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase { - private static final String BUCKET = "RepositoryS3JavaRestTest-bucket"; - private static final String BASE_PATH = "RepositoryS3JavaRestTest-base-path"; + private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed"))); + private static final String BUCKET = "RepositoryS3RestReloadCredentialsIT-bucket-" + HASHED_SEED; + private static final String BASE_PATH = "RepositoryS3RestReloadCredentialsIT-base-path-" + HASHED_SEED; public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored"); diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java index 0ae8af0989fa6..64cb3c3fd3a69 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java @@ -9,8 +9,8 @@ package org.elasticsearch.repositories.s3; +import fixture.aws.imds.Ec2ImdsHttpFixture; import fixture.s3.S3HttpFixture; -import fixture.s3.S3HttpFixtureWithEC2; import fixture.s3.S3HttpFixtureWithSessionToken; import com.carrotsearch.randomizedtesting.annotations.Name; @@ -18,6 +18,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -25,15 +26,34 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Set; + @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - public static final S3HttpFixture s3Fixture = new S3HttpFixture(); - public static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(); - public static final S3HttpFixtureWithEC2 s3Ec2 = new S3HttpFixtureWithEC2(); + private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed"))); + private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED; + private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED; + private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED; + + private static final S3HttpFixture s3Fixture = new S3HttpFixture(); + + private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken( + "session_token_bucket", + "session_token_base_path_integration_tests", + System.getProperty("s3TemporaryAccessKey"), + TEMPORARY_SESSION_TOKEN + ); + + private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken( + "ec2_bucket", + "ec2_base_path", + IMDS_ACCESS_KEY, + IMDS_SESSION_TOKEN + ); - private static final String s3TemporarySessionToken = "session_token"; + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of()); public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-s3") @@ -41,15 +61,19 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey")) .keystore("s3.client.integration_test_temporary.access_key", System.getProperty("s3TemporaryAccessKey")) .keystore("s3.client.integration_test_temporary.secret_key", System.getProperty("s3TemporarySecretKey")) - .keystore("s3.client.integration_test_temporary.session_token", s3TemporarySessionToken) + .keystore("s3.client.integration_test_temporary.session_token", TEMPORARY_SESSION_TOKEN) .setting("s3.client.integration_test_permanent.endpoint", s3Fixture::getAddress) .setting("s3.client.integration_test_temporary.endpoint", s3HttpFixtureWithSessionToken::getAddress) - .setting("s3.client.integration_test_ec2.endpoint", s3Ec2::getAddress) - .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", s3Ec2::getAddress) + .setting("s3.client.integration_test_ec2.endpoint", s3HttpFixtureWithImdsSessionToken::getAddress) + .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Ec2).around(s3HttpFixtureWithSessionToken).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture) + .around(s3HttpFixtureWithSessionToken) + .around(s3HttpFixtureWithImdsSessionToken) + .around(ec2ImdsHttpFixture) + .around(cluster); @ParametersFactory public static Iterable parameters() throws Exception { diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java index fa21797540c17..a522c9b17145b 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java @@ -9,28 +9,48 @@ package org.elasticsearch.repositories.s3; -import fixture.s3.S3HttpFixtureWithECS; +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.s3.S3HttpFixtureWithSessionToken; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Set; + public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - private static final S3HttpFixtureWithECS s3Ecs = new S3HttpFixtureWithECS(); + + private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed"))); + private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED; + private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED; + + private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken( + "ecs_bucket", + "ecs_base_path", + ECS_ACCESS_KEY, + ECS_SESSION_TOKEN + ); + + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + ECS_ACCESS_KEY, + ECS_SESSION_TOKEN, + Set.of("/ecs_credentials_endpoint") + ); public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-s3") - .setting("s3.client.integration_test_ecs.endpoint", s3Ecs::getAddress) - .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> (s3Ecs.getAddress() + "/ecs_credentials_endpoint")) + .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress) + .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> ec2ImdsHttpFixture.getAddress() + "/ecs_credentials_endpoint") .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Ecs).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(ec2ImdsHttpFixture).around(cluster); @ParametersFactory public static Iterable parameters() throws Exception { diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index b971a52b7afb6..36c860f1fb90b 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -33,6 +33,7 @@ import io.netty.handler.timeout.ReadTimeoutException; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.AttributeKey; +import io.netty.util.ResourceLeakDetector; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -410,6 +411,9 @@ protected Result beginEncode(HttpResponse httpResponse, String acceptEncoding) t } }); } + if (ResourceLeakDetector.isEnabled()) { + ch.pipeline().addLast(new Netty4LeakDetectionHandler()); + } ch.pipeline() .addLast( "pipelining", diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java new file mode 100644 index 0000000000000..8a0274872e493 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpRequest; + +import org.elasticsearch.tasks.Task; + +/** + * Inbound channel handler that enrich leaking buffers information from HTTP request. + * It helps to detect which handler is leaking buffers. Especially integration tests that run with + * paranoid leak detector that samples all buffers for leaking. Supplying informative opaque-id in + * integ test helps to narrow down problem (for example test name). + */ +public class Netty4LeakDetectionHandler extends ChannelInboundHandlerAdapter { + + private String info; + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + if (msg instanceof HttpRequest request) { + var opaqueId = request.headers().get(Task.X_OPAQUE_ID_HTTP_HEADER); + info = "method: " + request.method() + "; uri: " + request.uri() + "; x-opaque-id: " + opaqueId; + } + if (msg instanceof HttpContent content) { + content.touch(info); + } + ctx.fireChannelRead(msg); + } +} diff --git a/muted-tests.yml b/muted-tests.yml index f33ca972b7d36..37f36e9a19340 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -168,9 +168,6 @@ tests: - class: org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsCanMatchOnCoordinatorIntegTests method: testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQueryingAnyNodeWhenTheyAreOutsideOfTheQueryRange issue: https://github.com/elastic/elasticsearch/issues/116523 -- class: org.elasticsearch.xpack.logsdb.qa.StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT - method: testMatchAllQuery - issue: https://github.com/elastic/elasticsearch/issues/116536 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT method: test {categorize.Categorize} issue: https://github.com/elastic/elasticsearch/issues/116434 @@ -188,9 +185,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=snapshot/20_operator_privileges_disabled/Operator only settings can be set and restored by non-operator user when operator privileges is disabled} issue: https://github.com/elastic/elasticsearch/issues/116775 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set} - issue: https://github.com/elastic/elasticsearch/issues/116777 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT issue: https://github.com/elastic/elasticsearch/issues/116851 - class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT @@ -208,9 +202,6 @@ tests: - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testMultipleInferencesTriggeringDownloadAndDeploy issue: https://github.com/elastic/elasticsearch/issues/117208 -- class: org.elasticsearch.xpack.logsdb.qa.StandardVersusLogsStoredSourceChallengeRestIT - method: testEsqlSource - issue: https://github.com/elastic/elasticsearch/issues/117212 - class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT method: testEnterpriseDownloaderTask issue: https://github.com/elastic/elasticsearch/issues/115163 @@ -229,12 +220,26 @@ tests: - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT method: testInferDeploysDefaultElser issue: https://github.com/elastic/elasticsearch/issues/114913 -- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT - method: testCancelRequestWhenFailingFetchingPages - issue: https://github.com/elastic/elasticsearch/issues/117397 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint} + issue: https://github.com/elastic/elasticsearch/issues/117027 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint} + issue: https://github.com/elastic/elasticsearch/issues/117349 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/102992 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=transform/transforms_reset/Test reset running transform} + issue: https://github.com/elastic/elasticsearch/issues/117473 +- class: org.elasticsearch.xpack.esql.qa.single_node.FieldExtractorIT + method: testConstantKeywordField + issue: https://github.com/elastic/elasticsearch/issues/117524 +- class: org.elasticsearch.xpack.esql.qa.multi_node.FieldExtractorIT + method: testConstantKeywordField + issue: https://github.com/elastic/elasticsearch/issues/117524 +- class: org.elasticsearch.repositories.s3.RepositoryS3EcsClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/117525 # Examples: # diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index 090f409fd46d0..86a0151e33119 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.xcontent.XContentBuilder; @@ -417,9 +418,15 @@ public void testSyntheticSource() throws IOException { if (isOldCluster()) { Request createIndex = new Request("PUT", "/synthetic"); XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject(); + boolean useIndexSetting = SourceFieldMapper.onOrAfterDeprecateModeVersion(getOldClusterIndexVersion()); + if (useIndexSetting) { + indexSpec.startObject("settings").field("index.mapping.source.mode", "synthetic").endObject(); + } indexSpec.startObject("mappings"); { - indexSpec.startObject("_source").field("mode", "synthetic").endObject(); + if (useIndexSetting == false) { + indexSpec.startObject("_source").field("mode", "synthetic").endObject(); + } indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject(); } indexSpec.endObject(); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json b/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json index 5cd2b0e26459e..a7a7ebe838eab 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json @@ -65,6 +65,11 @@ "type":"boolean", "description":"Specify whether wildcard and prefix queries should be analyzed (default: false)" }, + "ccs_minimize_roundtrips":{ + "type":"boolean", + "default":false, + "description":"When doing a cross-cluster search, setting it to true may improve overall search latency, particularly when searching clusters with a large number of shards. However, when set to true, the progress of searches on the remote clusters will not be received until the search finishes on all clusters." + }, "default_operator":{ "type":"enum", "options":[ @@ -126,6 +131,16 @@ "type":"string", "description":"Specify the node or shard the operation should be performed on (default: random)" }, + "pre_filter_shard_size":{ + "type":"number", + "default": 1, + "description":"Cannot be changed: this is to enforce the execution of a pre-filter roundtrip to retrieve statistics from each shard so that the ones that surely don’t hold any document matching the query get skipped." + }, + "rest_total_hits_as_int":{ + "type":"boolean", + "description":"Indicates whether hits.total should be rendered as an integer or an object in the rest search response", + "default":false + }, "q":{ "type":"string", "description":"Query in the Lucene query string syntax" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml index 675b98133ce11..93f1fafa7ab85 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml @@ -17,7 +17,10 @@ setup: - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 - do: synonyms.get_synonym: @@ -64,7 +67,10 @@ setup: - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 - do: synonyms.get_synonym: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml index 4e77e10495109..7f545b466e65f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml @@ -14,7 +14,10 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml index 5e6d4ec2341ad..9e6af0f471e6e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml @@ -17,7 +17,10 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 --- "Get synonyms set": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml index 23c907f6a1137..62e8fe333ce99 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml @@ -15,7 +15,11 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + --- "Delete synonyms set": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml index 7c145dafd81cd..3815ea2c96c97 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml @@ -13,7 +13,10 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 - do: synonyms.put_synonym: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml index d8611000fe465..02757f711f690 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml @@ -17,7 +17,11 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + --- "Update a synonyms rule": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml index 0c962b51e08cb..9f1aa1d254169 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml @@ -17,8 +17,10 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true - + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 --- "Get a synonym rule": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml index 41ab293158a35..d2c706decf4fd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml @@ -17,7 +17,10 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 --- "Delete synonym rule": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml index 3aba0f0b4b78b..965cae551fab2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml @@ -16,7 +16,10 @@ setup: # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 # Create an index with synonym_filter that uses that synonyms set - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml index ac01f2dc0178a..d6c98673253fb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml @@ -28,7 +28,10 @@ # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. - do: cluster.health: - wait_for_no_initializing_shards: true + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 # Create my_index1 with synonym_filter that uses synonyms_set1 - do: diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 7a5f469a57fa1..6344aa2a72ca9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -131,6 +131,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion ADD_ROLE_MAPPING_CLEANUP_MIGRATION = def(8_518_00_0, Version.LUCENE_9_12_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT_BACKPORT = def(8_519_00_0, Version.LUCENE_9_12_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID_BACKPORT = def(8_520_00_0, Version.LUCENE_9_12_0); + public static final IndexVersion V8_DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index e5b12f748543f..e7c7ec3535b91 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; @@ -297,7 +298,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { return DEFAULT; } - if (c.indexVersionCreated().onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)) { + if (onOrAfterDeprecateModeVersion(c.indexVersionCreated())) { return resolveStaticInstance(settingSourceMode); } else { return new SourceFieldMapper(settingSourceMode, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, true); @@ -307,14 +308,14 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { c.getIndexSettings().getMode(), c.getSettings(), c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK), - c.indexVersionCreated().before(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER) + onOrAfterDeprecateModeVersion(c.indexVersionCreated()) == false ) ) { @Override public MetadataFieldMapper.Builder parse(String name, Map node, MappingParserContext parserContext) throws MapperParsingException { assert name.equals(SourceFieldMapper.NAME) : name; - if (parserContext.indexVersionCreated().after(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER) && node.containsKey("mode")) { + if (onOrAfterDeprecateModeVersion(parserContext.indexVersionCreated()) && node.containsKey("mode")) { deprecationLogger.critical(DeprecationCategory.MAPPINGS, "mapping_source_mode", SourceFieldMapper.DEPRECATION_WARNING); } return super.parse(name, node, parserContext); @@ -481,4 +482,9 @@ public boolean isDisabled() { public boolean isStored() { return mode == null || mode == Mode.STORED; } + + public static boolean onOrAfterDeprecateModeVersion(IndexVersion version) { + return version.onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER) + || version.between(IndexVersions.V8_DEPRECATE_SOURCE_MODE_MAPPER, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index e5c4826bfce97..794b30aa5aab2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -27,7 +27,7 @@ private SearchCapabilities() {} /** Support synthetic source with `bit` type in `dense_vector` field when `index` is set to `false`. */ private static final String BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY = "bit_dense_vector_synthetic_source"; /** Support Byte and Float with Bit dot product. */ - private static final String BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY = "byte_float_bit_dot_product"; + private static final String BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY = "byte_float_bit_dot_product_with_bugfix"; /** Support docvalue_fields parameter for `dense_vector` field. */ private static final String DENSE_VECTOR_DOCVALUE_FIELDS = "dense_vector_docvalue_fields"; /** Support transforming rank rrf queries to the corresponding rrf retriever. */ @@ -41,7 +41,7 @@ private SearchCapabilities() {} /** Support multi-dense-vector script field access. */ private static final String MULTI_DENSE_VECTOR_SCRIPT_ACCESS = "multi_dense_vector_script_access"; /** Initial support for multi-dense-vector maxSim functions access. */ - private static final String MULTI_DENSE_VECTOR_SCRIPT_MAX_SIM = "multi_dense_vector_script_max_sim"; + private static final String MULTI_DENSE_VECTOR_SCRIPT_MAX_SIM = "multi_dense_vector_script_max_sim_with_bugfix"; private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs"; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java index 0baecf6e3f92b..441b30f872a35 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -205,15 +205,7 @@ public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throw CompositeKey key = queue.toCompositeKey(slot); InternalAggregations aggs = subAggsForBuckets.apply(slot); long docCount = queue.getDocCount(slot); - buckets[(int) queue.size()] = new InternalComposite.InternalBucket( - sourceNames, - formats, - key, - reverseMuls, - missingOrders, - docCount, - aggs - ); + buckets[(int) queue.size()] = new InternalComposite.InternalBucket(sourceNames, formats, key, docCount, aggs); } CompositeKey lastBucket = num > 0 ? buckets[num - 1].getRawKey() : null; return new InternalAggregation[] { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java index 8b3253418bc23..faa953e77edd8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java @@ -19,7 +19,6 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; -import org.elasticsearch.search.aggregations.KeyComparable; import org.elasticsearch.search.aggregations.bucket.BucketReducer; import org.elasticsearch.search.aggregations.bucket.IteratorAndCurrent; import org.elasticsearch.search.aggregations.support.SamplingContext; @@ -103,7 +102,7 @@ public InternalComposite(StreamInput in) throws IOException { } this.reverseMuls = in.readIntArray(); this.missingOrders = in.readArray(MissingOrder::readFromStream, MissingOrder[]::new); - this.buckets = in.readCollectionAsList((input) -> new InternalBucket(input, sourceNames, formats, reverseMuls, missingOrders)); + this.buckets = in.readCollectionAsList((input) -> new InternalBucket(input, sourceNames, formats)); this.afterKey = in.readOptionalWriteable(CompositeKey::new); this.earlyTerminated = in.readBoolean(); } @@ -155,15 +154,7 @@ public InternalComposite create(List newBuckets) { @Override public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) { - return new InternalBucket( - prototype.sourceNames, - prototype.formats, - prototype.key, - prototype.reverseMuls, - prototype.missingOrders, - prototype.docCount, - aggregations - ); + return new InternalBucket(prototype.sourceNames, prototype.formats, prototype.key, prototype.docCount, aggregations); } public int getSize() { @@ -206,7 +197,7 @@ protected AggregatorReducer getLeaderReducer(AggregationReduceContext reduceCont private final PriorityQueue> pq = new PriorityQueue<>(size) { @Override protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) { - return a.current().compareKey(b.current()) < 0; + return a.current().compareKey(b.current(), reverseMuls, missingOrders) < 0; } }; private boolean earlyTerminated = false; @@ -227,7 +218,7 @@ public InternalAggregation get() { final List result = new ArrayList<>(); while (pq.size() > 0) { IteratorAndCurrent top = pq.top(); - if (lastBucket != null && top.current().compareKey(lastBucket) != 0) { + if (lastBucket != null && top.current().compareKey(lastBucket, reverseMuls, missingOrders) != 0) { InternalBucket reduceBucket = reduceBucket(buckets, reduceContext); buckets.clear(); result.add(reduceBucket); @@ -306,7 +297,7 @@ private InternalBucket reduceBucket(List buckets, AggregationRed final var reducedFormats = reducer.getProto().formats; final long docCount = reducer.getDocCount(); final InternalAggregations aggs = reducer.getAggregations(); - return new InternalBucket(sourceNames, reducedFormats, reducer.getProto().key, reverseMuls, missingOrders, docCount, aggs); + return new InternalBucket(sourceNames, reducedFormats, reducer.getProto().key, docCount, aggs); } } @@ -329,16 +320,11 @@ public int hashCode() { return Objects.hash(super.hashCode(), size, buckets, afterKey, Arrays.hashCode(reverseMuls), Arrays.hashCode(missingOrders)); } - public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket - implements - CompositeAggregation.Bucket, - KeyComparable { + public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements CompositeAggregation.Bucket { private final CompositeKey key; private final long docCount; private final InternalAggregations aggregations; - private final transient int[] reverseMuls; - private final transient MissingOrder[] missingOrders; private final transient List sourceNames; private final transient List formats; @@ -346,32 +332,20 @@ public static class InternalBucket extends InternalMultiBucketAggregation.Intern List sourceNames, List formats, CompositeKey key, - int[] reverseMuls, - MissingOrder[] missingOrders, long docCount, InternalAggregations aggregations ) { this.key = key; this.docCount = docCount; this.aggregations = aggregations; - this.reverseMuls = reverseMuls; - this.missingOrders = missingOrders; this.sourceNames = sourceNames; this.formats = formats; } - InternalBucket( - StreamInput in, - List sourceNames, - List formats, - int[] reverseMuls, - MissingOrder[] missingOrders - ) throws IOException { + InternalBucket(StreamInput in, List sourceNames, List formats) throws IOException { this.key = new CompositeKey(in); this.docCount = in.readVLong(); this.aggregations = InternalAggregations.readFrom(in); - this.reverseMuls = reverseMuls; - this.missingOrders = missingOrders; this.sourceNames = sourceNames; this.formats = formats; } @@ -444,8 +418,7 @@ List getFormats() { return formats; } - @Override - public int compareKey(InternalBucket other) { + int compareKey(InternalBucket other, int[] reverseMuls, MissingOrder[] missingOrders) { for (int i = 0; i < key.size(); i++) { if (key.get(i) == null) { if (other.key.get(i) == null) { @@ -470,8 +443,6 @@ InternalBucket finalizeSampling(SamplingContext samplingContext) { sourceNames, formats, key, - reverseMuls, - missingOrders, samplingContext.scaleUp(docCount), InternalAggregations.finalizeSampling(aggregations, samplingContext) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java index be36ab9d6eac1..a4108caaf4fc3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java @@ -133,5 +133,6 @@ public void testCreateDynamicMapperBuilderContext() throws IOException { assertEquals(ObjectMapper.Defaults.DYNAMIC, resultFromParserContext.getDynamic()); assertEquals(MapperService.MergeReason.MAPPING_UPDATE, resultFromParserContext.getMergeReason()); assertFalse(resultFromParserContext.isInNestedContext()); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index d7f33b9cdb3ba..fa173bc64518e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -52,7 +52,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck( "enabled", topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", false).endObject()), - topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject()) + topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject()), + dm -> {} ); checker.registerUpdateCheck( topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("enabled", true).endObject()), @@ -62,14 +63,18 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck( topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "stored").endObject()), topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject()), - dm -> assertTrue(dm.metadataMapper(SourceFieldMapper.class).isSynthetic()) + dm -> { + assertTrue(dm.metadataMapper(SourceFieldMapper.class).isSynthetic()); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); + } ); checker.registerConflictCheck("includes", b -> b.array("includes", "foo*")); checker.registerConflictCheck("excludes", b -> b.array("excludes", "foo*")); checker.registerConflictCheck( "mode", topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject()), - topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "stored").endObject()) + topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "stored").endObject()), + dm -> assertWarnings(SourceFieldMapper.DEPRECATION_WARNING) ); } @@ -206,13 +211,14 @@ public void testSyntheticDisabledNotSupported() { ) ); assertThat(e.getMessage(), containsString("Cannot set both [mode] and [enabled] parameters")); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } public void testSyntheticUpdates() throws Exception { MapperService mapperService = createMapperService(""" { "_doc" : { "_source" : { "mode" : "synthetic" } } } """); - + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); SourceFieldMapper mapper = mapperService.documentMapper().sourceMapper(); assertTrue(mapper.enabled()); assertTrue(mapper.isSynthetic()); @@ -220,6 +226,7 @@ public void testSyntheticUpdates() throws Exception { merge(mapperService, """ { "_doc" : { "_source" : { "mode" : "synthetic" } } } """); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); mapper = mapperService.documentMapper().sourceMapper(); assertTrue(mapper.enabled()); assertTrue(mapper.isSynthetic()); @@ -230,11 +237,15 @@ public void testSyntheticUpdates() throws Exception { Exception e = expectThrows(IllegalArgumentException.class, () -> merge(mapperService, """ { "_doc" : { "_source" : { "mode" : "stored" } } } """)); + assertThat(e.getMessage(), containsString("Cannot update parameter [mode] from [synthetic] to [stored]")); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); merge(mapperService, """ { "_doc" : { "_source" : { "mode" : "disabled" } } } """); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); + mapper = mapperService.documentMapper().sourceMapper(); assertFalse(mapper.enabled()); assertFalse(mapper.isSynthetic()); @@ -270,6 +281,7 @@ public void testSupportsNonDefaultParameterValues() throws IOException { topMapping(b -> b.startObject("_source").field("mode", randomBoolean() ? "synthetic" : "stored").endObject()) ).documentMapper().sourceMapper(); assertThat(sourceFieldMapper, notNullValue()); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } Exception e = expectThrows( MapperParsingException.class, @@ -301,6 +313,8 @@ public void testSupportsNonDefaultParameterValues() throws IOException { .documentMapper() .sourceMapper() ); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); + assertThat(e.getMessage(), containsString("Parameter [mode=disabled] is not allowed in source")); e = expectThrows( @@ -409,6 +423,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { ParsedDocument doc = docMapper.parse(source(b -> { b.field("field1", "value1"); })); assertNotNull(doc.rootDoc().getField("_recovery_source")); assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"field1\":\"value1\"}"))); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } { Settings settings = Settings.builder().put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false).build(); @@ -419,6 +434,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); assertNull(doc.rootDoc().getField("_recovery_source")); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } } @@ -613,6 +629,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException { ParsedDocument doc = docMapper.parse(source(b -> { b.field("@timestamp", "2012-02-13"); })); assertNotNull(doc.rootDoc().getField("_recovery_source")); assertThat(doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\"}"))); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } { Settings settings = Settings.builder() @@ -623,6 +640,7 @@ public void testRecoverySourceWithLogsCustom() throws IOException { DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source(b -> b.field("@timestamp", "2012-02-13"))); assertNull(doc.rootDoc().getField("_recovery_source")); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } } @@ -691,6 +709,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { doc.rootDoc().getField("_recovery_source").binaryValue(), equalTo(new BytesRef("{\"@timestamp\":\"2012-02-13\",\"field\":\"value1\"}")) ); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } { Settings settings = Settings.builder() @@ -704,6 +723,7 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { source("123", b -> b.field("@timestamp", "2012-02-13").field("field", randomAlphaOfLength(5)), null) ); assertNull(doc.rootDoc().getField("_recovery_source")); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java index a49d895f38f67..307bc26c44ba6 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xcontent.XContentType; @@ -114,6 +115,7 @@ public void testGetFromTranslogWithSyntheticSource() throws IOException { "mode": "synthetic" """; runGetFromTranslogWithOptions(docToIndex, sourceOptions, expectedFetchedSource, "\"long\"", 7L, true); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } public void testGetFromTranslogWithDenseVector() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java b/server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java index c4a1699181efc..f908f51170478 100644 --- a/server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/script/MultiVectorScoreScriptUtilsTests.java @@ -200,7 +200,7 @@ public void testBitMultiVectorClassBindingsDotProduct() throws IOException { function = new MaxSimDotProduct(scoreScript, floatQueryVector, fieldName); assertEquals( "maxSimDotProduct result is not equal to the expected value!", - 0.42f + 0f + 1f - 1f - 0.42f, + -1.4f + 0.42f + 0f + 1f - 1f, function.maxSimDotProduct(), 0.001 ); diff --git a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java index 6b2178310d17c..dcaa64ede9e89 100644 --- a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java @@ -267,7 +267,7 @@ public void testBitVectorClassBindingsDotProduct() throws IOException { function = new DotProduct(scoreScript, floatQueryVector, fieldName); assertEquals( "dotProduct result is not equal to the expected value!", - 0.42f + 0f + 1f - 1f - 0.42f, + -1.4f + 0.42f + 0f + 1f - 1f, function.dotProduct(), 0.001 ); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java index 5fb1d0e760afa..7e7ccb1d72e80 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java @@ -143,18 +143,10 @@ protected InternalComposite createTestInstance(String name, Map continue; } keys.add(key); - InternalComposite.InternalBucket bucket = new InternalComposite.InternalBucket( - sourceNames, - formats, - key, - reverseMuls, - missingOrders, - 1L, - aggregations - ); + InternalComposite.InternalBucket bucket = new InternalComposite.InternalBucket(sourceNames, formats, key, 1L, aggregations); buckets.add(bucket); } - Collections.sort(buckets, (o1, o2) -> o1.compareKey(o2)); + Collections.sort(buckets, (o1, o2) -> o1.compareKey(o2, reverseMuls, missingOrders)); CompositeKey lastBucket = buckets.size() > 0 ? buckets.get(buckets.size() - 1).getRawKey() : null; return new InternalComposite( name, @@ -191,8 +183,6 @@ protected InternalComposite mutateInstance(InternalComposite instance) { sourceNames, formats, createCompositeKey(), - reverseMuls, - missingOrders, randomLongBetween(1, 100), InternalAggregations.EMPTY ) diff --git a/settings.gradle b/settings.gradle index 333f8272447c2..7bf03263031f1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -87,6 +87,7 @@ List projects = [ 'server', 'test:framework', 'test:fixtures:azure-fixture', + 'test:fixtures:ec2-imds-fixture', 'test:fixtures:gcs-fixture', 'test:fixtures:hdfs-fixture', 'test:fixtures:krb5kdc-fixture', diff --git a/test/fixtures/ec2-imds-fixture/build.gradle b/test/fixtures/ec2-imds-fixture/build.gradle new file mode 100644 index 0000000000000..7ad194acbb8fd --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/build.gradle @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +apply plugin: 'elasticsearch.java' + +description = 'Fixture for emulating the Instance Metadata Service (IMDS) running in AWS EC2' + +dependencies { + api project(':server') + api("junit:junit:${versions.junit}") { + transitive = false + } + api project(':test:framework') +} diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java new file mode 100644 index 0000000000000..68f46d778018c --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package fixture.aws.imds; + +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import org.junit.rules.ExternalResource; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Objects; +import java.util.Set; + +public class Ec2ImdsHttpFixture extends ExternalResource { + + private HttpServer server; + + private final String accessKey; + private final String sessionToken; + private final Set alternativeCredentialsEndpoints; + + public Ec2ImdsHttpFixture(String accessKey, String sessionToken, Set alternativeCredentialsEndpoints) { + this.accessKey = accessKey; + this.sessionToken = sessionToken; + this.alternativeCredentialsEndpoints = alternativeCredentialsEndpoints; + } + + protected HttpHandler createHandler() { + return new Ec2ImdsHttpHandler(accessKey, sessionToken, alternativeCredentialsEndpoints); + } + + public String getAddress() { + return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort(); + } + + public void stop(int delay) { + server.stop(delay); + } + + protected void before() throws Throwable { + server = HttpServer.create(resolveAddress(), 0); + server.createContext("/", Objects.requireNonNull(createHandler())); + server.start(); + } + + @Override + protected void after() { + stop(0); + } + + private static InetSocketAddress resolveAddress() { + try { + return new InetSocketAddress(InetAddress.getByName("localhost"), 0); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } +} diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java new file mode 100644 index 0000000000000..04e5e83bddfa9 --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package fixture.aws.imds; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collection; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.test.ESTestCase.randomIdentifier; + +/** + * Minimal HTTP handler that emulates the EC2 IMDS server + */ +@SuppressForbidden(reason = "this test uses a HttpServer to emulate the EC2 IMDS endpoint") +public class Ec2ImdsHttpHandler implements HttpHandler { + + private static final String IMDS_SECURITY_CREDENTIALS_PATH = "/latest/meta-data/iam/security-credentials/"; + + private final String accessKey; + private final String sessionToken; + private final Set validCredentialsEndpoints = ConcurrentCollections.newConcurrentSet(); + + public Ec2ImdsHttpHandler(String accessKey, String sessionToken, Collection alternativeCredentialsEndpoints) { + this.accessKey = Objects.requireNonNull(accessKey); + this.sessionToken = Objects.requireNonNull(sessionToken); + this.validCredentialsEndpoints.addAll(alternativeCredentialsEndpoints); + } + + @Override + public void handle(final HttpExchange exchange) throws IOException { + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + try (exchange) { + final var path = exchange.getRequestURI().getPath(); + final var requestMethod = exchange.getRequestMethod(); + + if ("PUT".equals(requestMethod) && "/latest/api/token".equals(path)) { + // Reject IMDSv2 probe + exchange.sendResponseHeaders(RestStatus.METHOD_NOT_ALLOWED.getStatus(), -1); + return; + } + + if ("GET".equals(requestMethod)) { + if (path.equals(IMDS_SECURITY_CREDENTIALS_PATH)) { + final var profileName = randomIdentifier(); + validCredentialsEndpoints.add(IMDS_SECURITY_CREDENTIALS_PATH + profileName); + final byte[] response = profileName.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + return; + } else if (validCredentialsEndpoints.contains(path)) { + final byte[] response = Strings.format( + """ + { + "AccessKeyId": "%s", + "Expiration": "%s", + "RoleArn": "%s", + "SecretAccessKey": "%s", + "Token": "%s" + }""", + accessKey, + ZonedDateTime.now(Clock.systemUTC()).plusDays(1L).format(DateTimeFormatter.ISO_DATE_TIME), + randomIdentifier(), + randomIdentifier(), + sessionToken + ).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + return; + } + } + + ExceptionsHelper.maybeDieOnAnotherThread(new AssertionError("not supported: " + requestMethod + " " + path)); + } + } +} diff --git a/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java new file mode 100644 index 0000000000000..5d5cbfae3fa60 --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package fixture.aws.imds; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpPrincipal; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.Set; + +public class Ec2ImdsHttpHandlerTests extends ESTestCase { + + public void testImdsV1() throws IOException { + final var accessKey = randomIdentifier(); + final var sessionToken = randomIdentifier(); + + final var handler = new Ec2ImdsHttpHandler(accessKey, sessionToken, Set.of()); + + final var roleResponse = handleRequest(handler, "GET", "/latest/meta-data/iam/security-credentials/"); + assertEquals(RestStatus.OK, roleResponse.status()); + final var profileName = roleResponse.body().utf8ToString(); + assertTrue(Strings.hasText(profileName)); + + final var credentialsResponse = handleRequest(handler, "GET", "/latest/meta-data/iam/security-credentials/" + profileName); + assertEquals(RestStatus.OK, credentialsResponse.status()); + + final var responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), credentialsResponse.body().streamInput(), false); + assertEquals(Set.of("AccessKeyId", "Expiration", "RoleArn", "SecretAccessKey", "Token"), responseMap.keySet()); + assertEquals(accessKey, responseMap.get("AccessKeyId")); + assertEquals(sessionToken, responseMap.get("Token")); + } + + public void testImdsV2Disabled() { + assertEquals( + RestStatus.METHOD_NOT_ALLOWED, + handleRequest(new Ec2ImdsHttpHandler(randomIdentifier(), randomIdentifier(), Set.of()), "PUT", "/latest/api/token").status() + ); + } + + private record TestHttpResponse(RestStatus status, BytesReference body) {} + + private static TestHttpResponse handleRequest(Ec2ImdsHttpHandler handler, String method, String uri) { + final var httpExchange = new TestHttpExchange(method, uri, BytesArray.EMPTY, TestHttpExchange.EMPTY_HEADERS); + try { + handler.handle(httpExchange); + } catch (IOException e) { + fail(e); + } + assertNotEquals(0, httpExchange.getResponseCode()); + return new TestHttpResponse(RestStatus.fromCode(httpExchange.getResponseCode()), httpExchange.getResponseBodyContents()); + } + + private static class TestHttpExchange extends HttpExchange { + + private static final Headers EMPTY_HEADERS = new Headers(); + + private final String method; + private final URI uri; + private final BytesReference requestBody; + private final Headers requestHeaders; + + private final Headers responseHeaders = new Headers(); + private final BytesStreamOutput responseBody = new BytesStreamOutput(); + private int responseCode; + + TestHttpExchange(String method, String uri, BytesReference requestBody, Headers requestHeaders) { + this.method = method; + this.uri = URI.create(uri); + this.requestBody = requestBody; + this.requestHeaders = requestHeaders; + } + + @Override + public Headers getRequestHeaders() { + return requestHeaders; + } + + @Override + public Headers getResponseHeaders() { + return responseHeaders; + } + + @Override + public URI getRequestURI() { + return uri; + } + + @Override + public String getRequestMethod() { + return method; + } + + @Override + public HttpContext getHttpContext() { + return null; + } + + @Override + public void close() {} + + @Override + public InputStream getRequestBody() { + try { + return requestBody.streamInput(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public OutputStream getResponseBody() { + return responseBody; + } + + @Override + public void sendResponseHeaders(int rCode, long responseLength) { + this.responseCode = rCode; + } + + @Override + public InetSocketAddress getRemoteAddress() { + return null; + } + + @Override + public int getResponseCode() { + return responseCode; + } + + public BytesReference getResponseBodyContents() { + return responseBody.bytes(); + } + + @Override + public InetSocketAddress getLocalAddress() { + return null; + } + + @Override + public String getProtocol() { + return "HTTP/1.1"; + } + + @Override + public Object getAttribute(String name) { + return null; + } + + @Override + public void setAttribute(String name, Object value) { + fail("setAttribute not implemented"); + } + + @Override + public void setStreams(InputStream i, OutputStream o) { + fail("setStreams not implemented"); + } + + @Override + public HttpPrincipal getPrincipal() { + fail("getPrincipal not implemented"); + throw new UnsupportedOperationException("getPrincipal not implemented"); + } + } + +} diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java deleted file mode 100644 index d7048cbea6b8a..0000000000000 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEC2.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package fixture.s3; - -import com.sun.net.httpserver.HttpHandler; - -import org.elasticsearch.rest.RestStatus; - -import java.nio.charset.StandardCharsets; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Locale; - -public class S3HttpFixtureWithEC2 extends S3HttpFixtureWithSessionToken { - - private static final String EC2_PATH = "/latest/meta-data/iam/security-credentials/"; - private static final String EC2_PROFILE = "ec2Profile"; - - public S3HttpFixtureWithEC2() { - this(true); - } - - public S3HttpFixtureWithEC2(boolean enabled) { - this(enabled, "ec2_bucket", "ec2_base_path", "ec2_access_key", "ec2_session_token"); - } - - public S3HttpFixtureWithEC2(boolean enabled, String bucket, String basePath, String accessKey, String sessionToken) { - super(enabled, bucket, basePath, accessKey, sessionToken); - } - - @Override - protected HttpHandler createHandler() { - final HttpHandler delegate = super.createHandler(); - - return exchange -> { - final String path = exchange.getRequestURI().getPath(); - // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html - if ("GET".equals(exchange.getRequestMethod()) && path.startsWith(EC2_PATH)) { - if (path.equals(EC2_PATH)) { - final byte[] response = EC2_PROFILE.getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "text/plain"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - - } else if (path.equals(EC2_PATH + EC2_PROFILE)) { - final byte[] response = buildCredentialResponse(accessKey, sessionToken).getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - } - - final byte[] response = "unknown profile".getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "text/plain"); - exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - - } - delegate.handle(exchange); - }; - } - - protected static String buildCredentialResponse(final String ec2AccessKey, final String ec2SessionToken) { - return String.format(Locale.ROOT, """ - { - "AccessKeyId": "%s", - "Expiration": "%s", - "RoleArn": "arn", - "SecretAccessKey": "secret_access_key", - "Token": "%s" - }""", ec2AccessKey, ZonedDateTime.now().plusDays(1L).format(DateTimeFormatter.ISO_DATE_TIME), ec2SessionToken); - } -} diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java deleted file mode 100644 index d6266ea75dd3a..0000000000000 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithECS.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package fixture.s3; - -import com.sun.net.httpserver.HttpHandler; - -import org.elasticsearch.rest.RestStatus; - -import java.nio.charset.StandardCharsets; - -public class S3HttpFixtureWithECS extends S3HttpFixtureWithEC2 { - - public S3HttpFixtureWithECS() { - this(true); - } - - public S3HttpFixtureWithECS(boolean enabled) { - this(enabled, "ecs_bucket", "ecs_base_path", "ecs_access_key", "ecs_session_token"); - } - - public S3HttpFixtureWithECS(boolean enabled, String bucket, String basePath, String accessKey, String sessionToken) { - super(enabled, bucket, basePath, accessKey, sessionToken); - } - - @Override - protected HttpHandler createHandler() { - final HttpHandler delegate = super.createHandler(); - - return exchange -> { - // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html - if ("GET".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getPath().equals("/ecs_credentials_endpoint")) { - final byte[] response = buildCredentialResponse(accessKey, sessionToken).getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); - return; - } - delegate.handle(exchange); - }; - } -} diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java index 1a1cbba651e06..001cc34d9b20d 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithSessionToken.java @@ -18,16 +18,8 @@ public class S3HttpFixtureWithSessionToken extends S3HttpFixture { protected final String sessionToken; - public S3HttpFixtureWithSessionToken() { - this(true); - } - - public S3HttpFixtureWithSessionToken(boolean enabled) { - this(enabled, "session_token_bucket", "session_token_base_path_integration_tests", "session_token_access_key", "session_token"); - } - - public S3HttpFixtureWithSessionToken(boolean enabled, String bucket, String basePath, String accessKey, String sessionToken) { - super(enabled, bucket, basePath, accessKey); + public S3HttpFixtureWithSessionToken(String bucket, String basePath, String accessKey, String sessionToken) { + super(true, bucket, basePath, accessKey); this.sessionToken = sessionToken; } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java index e86cb8562537f..449ecc099412f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MetadataMapperTestCase.java @@ -38,7 +38,7 @@ protected boolean isSupportedOn(IndexVersion version) { protected abstract void registerParameters(ParameterChecker checker) throws IOException; - private record ConflictCheck(XContentBuilder init, XContentBuilder update) {} + private record ConflictCheck(XContentBuilder init, XContentBuilder update, Consumer check) {} private record UpdateCheck(XContentBuilder init, XContentBuilder update, Consumer check) {} @@ -58,7 +58,7 @@ public void registerConflictCheck(String param, CheckedConsumer {})); } /** @@ -68,8 +68,8 @@ public void registerConflictCheck(String param, CheckedConsumer check) { + conflictChecks.put(param, new ConflictCheck(init, update, check)); } public void registerUpdateCheck(XContentBuilder init, XContentBuilder update, Consumer check) { @@ -95,6 +95,7 @@ public final void testUpdates() throws IOException { e.getMessage(), anyOf(containsString("Cannot update parameter [" + param + "]"), containsString("different [" + param + "]")) ); + checker.conflictChecks.get(param).check.accept(mapperService.documentMapper()); } for (UpdateCheck updateCheck : checker.updateChecks) { MapperService mapperService = createMapperService(updateCheck.init); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 8ca9c0709b359..bdef0ba631b72 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -112,7 +112,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -1835,9 +1834,10 @@ public static CreateIndexResponse createIndex(RestClient client, String name, Se if (settings != null && settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) { expectSoftDeletesWarning(request, name); - } else if (isSyntheticSourceConfiguredInMapping(mapping)) { - request.setOptions(expectVersionSpecificWarnings(v -> v.compatible(SourceFieldMapper.DEPRECATION_WARNING))); - } + } else if (isSyntheticSourceConfiguredInMapping(mapping) + && SourceFieldMapper.onOrAfterDeprecateModeVersion(minimumIndexVersion())) { + request.setOptions(expectVersionSpecificWarnings(v -> v.current(SourceFieldMapper.DEPRECATION_WARNING))); + } final Response response = client.performRequest(request); try (var parser = responseAsParser(response)) { return TestResponseParsers.parseCreateIndexResponse(parser); @@ -1898,8 +1898,30 @@ protected static boolean isSyntheticSourceConfiguredInMapping(String mapping) { if (sourceMapper == null) { return false; } - Object mode = sourceMapper.get("mode"); - return mode != null && mode.toString().toLowerCase(Locale.ROOT).equals("synthetic"); + return sourceMapper.get("mode") != null; + } + + @SuppressWarnings("unchecked") + protected static boolean isSyntheticSourceConfiguredInTemplate(String template) { + if (template == null) { + return false; + } + var values = XContentHelper.convertToMap(JsonXContent.jsonXContent, template, false); + for (Object value : values.values()) { + Map mappings = (Map) ((Map) value).get("mappings"); + if (mappings == null) { + continue; + } + Map sourceMapper = (Map) mappings.get(SourceFieldMapper.NAME); + if (sourceMapper == null) { + continue; + } + Object mode = sourceMapper.get("mode"); + if (mode != null) { + return true; + } + } + return false; } protected static Map getIndexSettings(String index) throws IOException { diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml index af28cbb7415a0..660db3a6b0e2e 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml @@ -5,8 +5,6 @@ _meta: managed: true template: mappings: - _source: - mode: synthetic properties: processor.event: type: constant_keyword diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml index 819d5d7eafb8e..d8fc13bce79b1 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml @@ -5,10 +5,12 @@ _meta: managed: true template: settings: - codec: best_compression - mapping: - # apm@settings sets `ignore_malformed: true`, but we need - # to disable this for metrics since they use synthetic source, - # and this combination is incompatible with the - # aggregate_metric_double field type. - ignore_malformed: false + index: + codec: best_compression + mapping: + # apm@settings sets `ignore_malformed: true`, but we need + # to disable this for metrics since they use synthetic source, + # and this combination is incompatible with the + # aggregate_metric_double field type. + ignore_malformed: false + source.mode: synthetic diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index fa209cdec3695..9484f577583eb 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 11 +version: 12 component-templates: # Data lifecycle. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/Cron.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/Cron.java index b9d39aa665848..c94b90b6c0c23 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/Cron.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/Cron.java @@ -8,11 +8,15 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.temporal.ChronoField; import java.util.Calendar; import java.util.Iterator; import java.util.Locale; @@ -232,6 +236,8 @@ public class Cron implements ToXContentFragment { private final String expression; + private ZoneId timeZone; + private transient TreeSet seconds; private transient TreeSet minutes; private transient TreeSet hours; @@ -246,7 +252,20 @@ public class Cron implements ToXContentFragment { private transient boolean nearestWeekday = false; private transient int lastdayOffset = 0; - public static final int MAX_YEAR = Calendar.getInstance(UTC, Locale.ROOT).get(Calendar.YEAR) + 100; + // Restricted to 50 years as the tzdb only has correct DST transition information for countries using a lunar calendar + // for the next ~60 years + public static final int MAX_YEAR = Calendar.getInstance(UTC, Locale.ROOT).get(Calendar.YEAR) + 50; + + public Cron(String expression, ZoneId timeZone) { + this.timeZone = timeZone; + assert expression != null : "cron expression cannot be null"; + this.expression = expression.toUpperCase(Locale.ROOT); + try { + buildExpression(this.expression); + } catch (Exception e) { + throw illegalArgument("invalid cron expression [{}]", e, expression); + } + } /** * Constructs a new CronExpression based on the specified @@ -259,13 +278,7 @@ public class Cron implements ToXContentFragment { * CronExpression */ public Cron(String expression) { - assert expression != null : "cron expression cannot be null"; - this.expression = expression.toUpperCase(Locale.ROOT); - try { - buildExpression(this.expression); - } catch (Exception e) { - throw illegalArgument("invalid cron expression [{}]", e, expression); - } + this(expression, UTC.toZoneId()); } /** @@ -275,7 +288,11 @@ public Cron(String expression) { * @param cron The existing cron expression to be copied */ public Cron(Cron cron) { - this(cron.expression); + this(cron.expression, cron.timeZone); + } + + public void setTimeZone(ZoneId timeZone) { + this.timeZone = timeZone; } /** @@ -286,31 +303,25 @@ public Cron(Cron cron) { * a time that is previous to the given time) * @return the next valid time (since the epoch) */ + @SuppressForbidden(reason = "In this case, the DST ambiguity of the atZone method is desired, understood and tested") public long getNextValidTimeAfter(final long time) { - // Computation is based on Gregorian year only. - Calendar cl = new java.util.GregorianCalendar(UTC, Locale.ROOT); - - // move ahead one second, since we're computing the time *after* the - // given time - final long afterTime = time + 1000; - // CronTrigger does not deal with milliseconds - cl.setTimeInMillis(afterTime); - cl.set(Calendar.MILLISECOND, 0); + LocalDateTime afterTimeLdt = LocalDateTime.ofInstant(java.time.Instant.ofEpochMilli(time), timeZone).plusSeconds(1); + LocalDateTimeLegacyWrapper cl = new LocalDateTimeLegacyWrapper(afterTimeLdt.with(ChronoField.MILLI_OF_SECOND, 0)); boolean gotOne = false; // loop until we've computed the next time, or we've past the endTime while (gotOne == false) { - if (cl.get(Calendar.YEAR) > 2999) { // prevent endless loop... + if (cl.getYear() > 2999) { // prevent endless loop... return -1; } SortedSet st = null; int t = 0; - int sec = cl.get(Calendar.SECOND); - int min = cl.get(Calendar.MINUTE); + int sec = cl.getSecond(); + int min = cl.getMinute(); // get second................................................. st = seconds.tailSet(sec); @@ -319,12 +330,12 @@ public long getNextValidTimeAfter(final long time) { } else { sec = seconds.first(); min++; - cl.set(Calendar.MINUTE, min); + cl.setMinute(min); } - cl.set(Calendar.SECOND, sec); + cl.setSecond(sec); - min = cl.get(Calendar.MINUTE); - int hr = cl.get(Calendar.HOUR_OF_DAY); + min = cl.getMinute(); + int hr = cl.getHour(); t = -1; // get minute................................................. @@ -337,15 +348,15 @@ public long getNextValidTimeAfter(final long time) { hr++; } if (min != t) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, min); - setCalendarHour(cl, hr); + cl.setSecond(0); + cl.setMinute(min); + cl.setHour(hr); continue; } - cl.set(Calendar.MINUTE, min); + cl.setMinute(min); - hr = cl.get(Calendar.HOUR_OF_DAY); - int day = cl.get(Calendar.DAY_OF_MONTH); + hr = cl.getHour(); + int day = cl.getDayOfMonth(); t = -1; // get hour................................................... @@ -358,16 +369,16 @@ public long getNextValidTimeAfter(final long time) { day++; } if (hr != t) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.DAY_OF_MONTH, day); - setCalendarHour(cl, hr); + cl.setSecond(0); + cl.setMinute(0); + cl.setDayOfMonth(day); + cl.setHour(hr); continue; } - cl.set(Calendar.HOUR_OF_DAY, hr); + cl.setHour(hr); - day = cl.get(Calendar.DAY_OF_MONTH); - int mon = cl.get(Calendar.MONTH) + 1; + day = cl.getDayOfMonth(); + int mon = cl.getMonth() + 1; // '+ 1' because calendar is 0-based for this field, and we are // 1-based t = -1; @@ -381,32 +392,32 @@ public long getNextValidTimeAfter(final long time) { if (lastdayOfMonth) { if (nearestWeekday == false) { t = day; - day = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); + day = getLastDayOfMonth(mon, cl.getYear()); day -= lastdayOffset; if (t > day) { mon++; if (mon > 12) { mon = 1; tmon = 3333; // ensure test of mon != tmon further below fails - cl.add(Calendar.YEAR, 1); + cl.plusYears(1); } day = 1; } } else { t = day; - day = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); + day = getLastDayOfMonth(mon, cl.getYear()); day -= lastdayOffset; - Calendar tcal = Calendar.getInstance(UTC, Locale.ROOT); - tcal.set(Calendar.SECOND, 0); - tcal.set(Calendar.MINUTE, 0); - tcal.set(Calendar.HOUR_OF_DAY, 0); - tcal.set(Calendar.DAY_OF_MONTH, day); - tcal.set(Calendar.MONTH, mon - 1); - tcal.set(Calendar.YEAR, cl.get(Calendar.YEAR)); + LocalDateTimeLegacyWrapper tcal = new LocalDateTimeLegacyWrapper(LocalDateTime.now(timeZone)); + tcal.setSecond(0); + tcal.setMinute(0); + tcal.setHour(0); + tcal.setDayOfMonth(day); + tcal.setMonth(mon - 1); + tcal.setYear(cl.getYear()); - int ldom = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); - int dow = tcal.get(Calendar.DAY_OF_WEEK); + int ldom = getLastDayOfMonth(mon, cl.getYear()); + int dow = tcal.getDayOfWeek(); if (dow == Calendar.SATURDAY && day == 1) { day += 2; @@ -418,13 +429,12 @@ public long getNextValidTimeAfter(final long time) { day += 1; } - tcal.set(Calendar.SECOND, sec); - tcal.set(Calendar.MINUTE, min); - tcal.set(Calendar.HOUR_OF_DAY, hr); - tcal.set(Calendar.DAY_OF_MONTH, day); - tcal.set(Calendar.MONTH, mon - 1); - long nTime = tcal.getTimeInMillis(); - if (nTime < afterTime) { + tcal.setSecond(sec); + tcal.setMinute(min); + tcal.setHour(hr); + tcal.setDayOfMonth(day); + tcal.setMonth(mon - 1); + if (tcal.isBefore(afterTimeLdt)) { day = 1; mon++; } @@ -433,16 +443,16 @@ public long getNextValidTimeAfter(final long time) { t = day; day = daysOfMonth.first(); - Calendar tcal = Calendar.getInstance(UTC, Locale.ROOT); - tcal.set(Calendar.SECOND, 0); - tcal.set(Calendar.MINUTE, 0); - tcal.set(Calendar.HOUR_OF_DAY, 0); - tcal.set(Calendar.DAY_OF_MONTH, day); - tcal.set(Calendar.MONTH, mon - 1); - tcal.set(Calendar.YEAR, cl.get(Calendar.YEAR)); + LocalDateTimeLegacyWrapper tcal = new LocalDateTimeLegacyWrapper(LocalDateTime.now(timeZone)); + tcal.setSecond(0); + tcal.setMinute(0); + tcal.setHour(0); + tcal.setDayOfMonth(day); + tcal.setMonth(mon - 1); + tcal.setYear(cl.getYear()); - int ldom = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); - int dow = tcal.get(Calendar.DAY_OF_WEEK); + int ldom = getLastDayOfMonth(mon, cl.getYear()); + int dow = tcal.getDayOfWeek(); if (dow == Calendar.SATURDAY && day == 1) { day += 2; @@ -454,13 +464,12 @@ public long getNextValidTimeAfter(final long time) { day += 1; } - tcal.set(Calendar.SECOND, sec); - tcal.set(Calendar.MINUTE, min); - tcal.set(Calendar.HOUR_OF_DAY, hr); - tcal.set(Calendar.DAY_OF_MONTH, day); - tcal.set(Calendar.MONTH, mon - 1); - long nTime = tcal.getTimeInMillis(); - if (nTime < afterTime) { + tcal.setSecond(sec); + tcal.setMinute(min); + tcal.setHour(hr); + tcal.setDayOfMonth(day); + tcal.setMonth(mon - 1); + if (tcal.isAfter(afterTimeLdt)) { day = daysOfMonth.first(); mon++; } @@ -468,7 +477,7 @@ public long getNextValidTimeAfter(final long time) { t = day; day = st.first(); // make sure we don't over-run a short month, such as february - int lastDay = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); + int lastDay = getLastDayOfMonth(mon, cl.getYear()); if (day > lastDay) { day = daysOfMonth.first(); mon++; @@ -479,11 +488,11 @@ public long getNextValidTimeAfter(final long time) { } if (day != t || mon != tmon) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, day); - cl.set(Calendar.MONTH, mon - 1); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(day); + cl.setMonth(mon - 1); // '- 1' because calendar is 0-based for this field, and we // are 1-based continue; @@ -493,7 +502,7 @@ public long getNextValidTimeAfter(final long time) { // the month? int dow = daysOfWeek.first(); // desired // d-o-w - int cDow = cl.get(Calendar.DAY_OF_WEEK); // current d-o-w + int cDow = cl.getDayOfWeek(); // current d-o-w int daysToAdd = 0; if (cDow < dow) { daysToAdd = dow - cDow; @@ -502,15 +511,15 @@ public long getNextValidTimeAfter(final long time) { daysToAdd = dow + (7 - cDow); } - int lDay = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); + int lDay = getLastDayOfMonth(mon, cl.getYear()); if (day + daysToAdd > lDay) { // did we already miss the // last one? - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, 1); - cl.set(Calendar.MONTH, mon); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(1); + cl.setMonth(mon); // no '- 1' here because we are promoting the month continue; } @@ -523,11 +532,11 @@ public long getNextValidTimeAfter(final long time) { day += daysToAdd; if (daysToAdd > 0) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, day); - cl.set(Calendar.MONTH, mon - 1); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(day); + cl.setMonth(mon - 1); // '- 1' here because we are not promoting the month continue; } @@ -536,7 +545,7 @@ public long getNextValidTimeAfter(final long time) { // are we looking for the Nth XXX day in the month? int dow = daysOfWeek.first(); // desired // d-o-w - int cDow = cl.get(Calendar.DAY_OF_WEEK); // current d-o-w + int cDow = cl.getDayOfWeek(); // current d-o-w int daysToAdd = 0; if (cDow < dow) { daysToAdd = dow - cDow; @@ -557,25 +566,25 @@ public long getNextValidTimeAfter(final long time) { daysToAdd = (nthdayOfWeek - weekOfMonth) * 7; day += daysToAdd; - if (daysToAdd < 0 || day > getLastDayOfMonth(mon, cl.get(Calendar.YEAR))) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, 1); - cl.set(Calendar.MONTH, mon); + if (daysToAdd < 0 || day > getLastDayOfMonth(mon, cl.getYear())) { + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(1); + cl.setMonth(mon); // no '- 1' here because we are promoting the month continue; } else if (daysToAdd > 0 || dayShifted) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, day); - cl.set(Calendar.MONTH, mon - 1); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(day); + cl.setMonth(mon - 1); // '- 1' here because we are NOT promoting the month continue; } } else { - int cDow = cl.get(Calendar.DAY_OF_WEEK); // current d-o-w + int cDow = cl.getDayOfWeek(); // current d-o-w int dow = daysOfWeek.first(); // desired // d-o-w st = daysOfWeek.tailSet(cDow); @@ -591,23 +600,23 @@ public long getNextValidTimeAfter(final long time) { daysToAdd = dow + (7 - cDow); } - int lDay = getLastDayOfMonth(mon, cl.get(Calendar.YEAR)); + int lDay = getLastDayOfMonth(mon, cl.getYear()); if (day + daysToAdd > lDay) { // will we pass the end of // the month? - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, 1); - cl.set(Calendar.MONTH, mon); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(1); + cl.setMonth(mon); // no '- 1' here because we are promoting the month continue; } else if (daysToAdd > 0) { // are we swithing days? - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, day + daysToAdd); - cl.set(Calendar.MONTH, mon - 1); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(day + daysToAdd); + cl.setMonth(mon - 1); // '- 1' because calendar is 0-based for this field, // and we are 1-based continue; @@ -618,12 +627,12 @@ public long getNextValidTimeAfter(final long time) { // throw new UnsupportedOperationException( // "Support for specifying both a day-of-week AND a day-of-month parameter is not implemented."); } - cl.set(Calendar.DAY_OF_MONTH, day); + cl.setDayOfMonth(day); - mon = cl.get(Calendar.MONTH) + 1; + mon = cl.getMonth() + 1; // '+ 1' because calendar is 0-based for this field, and we are // 1-based - int year = cl.get(Calendar.YEAR); + int year = cl.getYear(); t = -1; // test for expressions that never generate a valid fire date, @@ -643,21 +652,21 @@ public long getNextValidTimeAfter(final long time) { year++; } if (mon != t) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, 1); - cl.set(Calendar.MONTH, mon - 1); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(1); + cl.setMonth(mon - 1); // '- 1' because calendar is 0-based for this field, and we are // 1-based - cl.set(Calendar.YEAR, year); + cl.setYear(year); continue; } - cl.set(Calendar.MONTH, mon - 1); + cl.setMonth(mon - 1); // '- 1' because calendar is 0-based for this field, and we are // 1-based - year = cl.get(Calendar.YEAR); + year = cl.getYear(); t = -1; // get year................................................... @@ -671,22 +680,24 @@ public long getNextValidTimeAfter(final long time) { } if (year != t) { - cl.set(Calendar.SECOND, 0); - cl.set(Calendar.MINUTE, 0); - cl.set(Calendar.HOUR_OF_DAY, 0); - cl.set(Calendar.DAY_OF_MONTH, 1); - cl.set(Calendar.MONTH, 0); + cl.setSecond(0); + cl.setMinute(0); + cl.setHour(0); + cl.setDayOfMonth(1); + cl.setMonth(0); // '- 1' because calendar is 0-based for this field, and we are // 1-based - cl.set(Calendar.YEAR, year); + cl.setYear(year); continue; } - cl.set(Calendar.YEAR, year); + cl.setYear(year); gotOne = true; } // while( done == false ) - return cl.getTimeInMillis(); + LocalDateTime nextRuntime = cl.getLocalDateTime(); + + return nextRuntime.atZone(timeZone).toInstant().toEpochMilli(); } public String expression() { @@ -735,7 +746,7 @@ public String getExpressionSummary() { @Override public int hashCode() { - return Objects.hash(expression); + return Objects.hash(expression, timeZone); } @Override @@ -747,7 +758,7 @@ public boolean equals(Object obj) { return false; } final Cron other = (Cron) obj; - return Objects.equals(this.expression, other.expression); + return Objects.equals(this.expression, other.expression) && Objects.equals(this.timeZone, other.timeZone); } /** @@ -757,7 +768,7 @@ public boolean equals(Object obj) { */ @Override public String toString() { - return expression; + return "Cron{" + "timeZone=" + timeZone + ", expression='" + expression + '\'' + '}'; } /** @@ -1430,7 +1441,7 @@ private static int getLastDayOfMonth(int monthNum, int year) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.value(toString()); + return builder.value(expression); } private static class ValueSet { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/LocalDateTimeLegacyWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/LocalDateTimeLegacyWrapper.java new file mode 100644 index 0000000000000..e540acc8042eb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/scheduler/LocalDateTimeLegacyWrapper.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.scheduler; + +import java.time.LocalDateTime; +import java.time.chrono.ChronoLocalDateTime; + +/** + * This class is designed to wrap the LocalDateTime class in order to make it behave, in terms of mutation, like a legacy Calendar class. + * This is to provide compatibility with the existing Cron next runtime calculation algorithm which relies on certain quirks of the Calendar + * such as days of the week being numbered starting on Sunday==1 and being able to set the current hour to 24 and have it roll over to + * midnight the next day. + */ +public class LocalDateTimeLegacyWrapper { + + private LocalDateTime ldt; + + public LocalDateTimeLegacyWrapper(LocalDateTime ldt) { + this.ldt = ldt; + } + + public int getYear() { + return ldt.getYear(); + } + + public int getDayOfMonth() { + return ldt.getDayOfMonth(); + } + + public int getHour() { + return ldt.getHour(); + } + + public int getMinute() { + return ldt.getMinute(); + } + + public int getSecond() { + return ldt.getSecond(); + } + + public int getDayOfWeek() { + return (ldt.getDayOfWeek().getValue() % 7) + 1; + } + + public int getMonth() { + return ldt.getMonthValue() - 1; + } + + public void setYear(int year) { + ldt = ldt.withYear(year); + } + + public void setDayOfMonth(int dayOfMonth) { + var lengthOfMonth = ldt.getMonth().length(ldt.toLocalDate().isLeapYear()); + if (dayOfMonth <= lengthOfMonth) { + ldt = ldt.withDayOfMonth(dayOfMonth); + } else { + var months = dayOfMonth / lengthOfMonth; + var day = dayOfMonth % lengthOfMonth; + ldt = ldt.plusMonths(months).withDayOfMonth(day); + } + } + + public void setMonth(int month) { + month++; // Months are 0-based in Calendar + if (month <= 12) { + ldt = ldt.withMonth(month); + } else { + var years = month / 12; + var monthOfYear = month % 12; + ldt = ldt.plusYears(years).withMonth(monthOfYear); + } + } + + public void setHour(int hour) { + if (hour < 24) { + ldt = ldt.withHour(hour); + } else { + var days = hour / 24; + var hourOfDay = hour % 24; + ldt = ldt.plusDays(days).withHour(hourOfDay); + } + } + + public void setMinute(int minute) { + if (minute < 60) { + ldt = ldt.withMinute(minute); + } else { + var hours = minute / 60; + var minuteOfHour = minute % 60; + ldt = ldt.plusHours(hours).withMinute(minuteOfHour); + } + } + + public void setSecond(int second) { + if (second < 60) { + ldt = ldt.withSecond(second); + } else { + var minutes = second / 60; + var secondOfMinute = second % 60; + ldt = ldt.plusMinutes(minutes).withSecond(secondOfMinute); + } + } + + public void plusYears(long years) { + ldt = ldt.plusYears(years); + } + + public void plusSeconds(long seconds) { + ldt = ldt.plusSeconds(seconds); + } + + public boolean isAfter(ChronoLocalDateTime other) { + return ldt.isAfter(other); + } + + public boolean isBefore(ChronoLocalDateTime other) { + return ldt.isBefore(other); + } + + public LocalDateTime getLocalDateTime() { + return ldt; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/scheduler/CronTimezoneTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/scheduler/CronTimezoneTests.java new file mode 100644 index 0000000000000..1e469002457d8 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/scheduler/CronTimezoneTests.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.scheduler; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; + +import static java.time.Instant.ofEpochMilli; +import static java.util.TimeZone.getTimeZone; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; + +public class CronTimezoneTests extends ESTestCase { + + public void testForFixedOffsetCorrectlyCalculateNextRuntime() { + Cron cron = new Cron("0 0 2 * * ?", ZoneOffset.of("+1")); + long midnightUTC = Instant.parse("2020-01-01T00:00:00Z").toEpochMilli(); + long nextValidTimeAfter = cron.getNextValidTimeAfter(midnightUTC); + assertThat(Instant.ofEpochMilli(nextValidTimeAfter), equalTo(Instant.parse("2020-01-01T01:00:00Z"))); + } + + public void testForLondonFixedDSTTransitionCheckCorrectSchedule() { + ZoneId londonZone = getTimeZone("Europe/London").toZoneId(); + + Cron cron = new Cron("0 0 2 * * ?", londonZone); + ZoneRules londonZoneRules = londonZone.getRules(); + Instant springMidnight = Instant.parse("2020-03-01T00:00:00Z"); + long timeBeforeDST = springMidnight.toEpochMilli(); + + assertThat(cron.getNextValidTimeAfter(timeBeforeDST), equalTo(Instant.parse("2020-03-01T02:00:00Z").toEpochMilli())); + + ZoneOffsetTransition zoneOffsetTransition = londonZoneRules.nextTransition(springMidnight); + + Instant timeAfterDST = zoneOffsetTransition.getDateTimeBefore() + .plusDays(1) + .atZone(ZoneOffset.UTC) + .withHour(0) + .withMinute(0) + .toInstant(); + + assertThat(cron.getNextValidTimeAfter(timeAfterDST.toEpochMilli()), equalTo(Instant.parse("2020-03-30T01:00:00Z").toEpochMilli())); + } + + public void testRandomDSTTransitionCalculateNextTimeCorrectlyRelativeToUTC() { + ZoneId timeZone = generateRandomDSTZone(); + + logger.info("Testing for timezone {}", timeZone); + + ZoneOffsetTransition zoneOffsetTransition = timeZone.getRules().nextTransition(Instant.now()); + + ZonedDateTime midnightBefore = zoneOffsetTransition.getDateTimeBefore().atZone(timeZone).minusDays(2).withHour(0).withMinute(0); + ZonedDateTime midnightAfter = zoneOffsetTransition.getDateTimeAfter().atZone(timeZone).plusDays(2).withHour(0).withMinute(0); + + long epochBefore = midnightBefore.toInstant().toEpochMilli(); + long epochAfter = midnightAfter.toInstant().toEpochMilli(); + + Cron cron = new Cron("0 0 2 * * ?", timeZone); + + long nextScheduleBefore = cron.getNextValidTimeAfter(epochBefore); + long nextScheduleAfter = cron.getNextValidTimeAfter(epochAfter); + + assertThat(nextScheduleBefore - epochBefore, equalTo(2 * 60 * 60 * 1000L)); // 2 hours + assertThat(nextScheduleAfter - epochAfter, equalTo(2 * 60 * 60 * 1000L)); // 2 hours + + ZonedDateTime utcMidnightBefore = zoneOffsetTransition.getDateTimeBefore() + .atZone(ZoneOffset.UTC) + .minusDays(2) + .withHour(0) + .withMinute(0); + + ZonedDateTime utcMidnightAfter = zoneOffsetTransition.getDateTimeAfter() + .atZone(ZoneOffset.UTC) + .plusDays(2) + .withHour(0) + .withMinute(0); + + long utcEpochBefore = utcMidnightBefore.toInstant().toEpochMilli(); + long utcEpochAfter = utcMidnightAfter.toInstant().toEpochMilli(); + + long nextUtcScheduleBefore = cron.getNextValidTimeAfter(utcEpochBefore); + long nextUtcScheduleAfter = cron.getNextValidTimeAfter(utcEpochAfter); + + assertThat(nextUtcScheduleBefore - utcEpochBefore, not(equalTo(nextUtcScheduleAfter - utcEpochAfter))); + + } + + private ZoneId generateRandomDSTZone() { + ZoneId timeZone; + int i = 0; + boolean found; + do { + timeZone = randomZone(); + found = getTimeZone(timeZone).useDaylightTime(); + i++; + } while (found == false && i <= 500); // Infinite loop prevention + + if (found == false) { + fail("Could not find a timezone with DST"); + } + + logger.debug("Testing for timezone {} after {} iterations", timeZone, i); + return timeZone; + } + + public void testForGMTGapTransitionTriggerTimeIsAsIfTransitionHasntHappenedYet() { + ZoneId london = ZoneId.of("Europe/London"); + Cron cron = new Cron("0 30 1 * * ?", london); // Every day at 1:30 + + Instant beforeTransition = Instant.parse("2025-03-30T00:00:00Z"); + long beforeTransitionEpoch = beforeTransition.toEpochMilli(); + + long nextValidTimeAfter = cron.getNextValidTimeAfter(beforeTransitionEpoch); + assertThat(ofEpochMilli(nextValidTimeAfter), equalTo(Instant.parse("2025-03-30T01:30:00Z"))); + } + + public void testForGMTOverlapTransitionTriggerSkipSecondExecution() { + ZoneId london = ZoneId.of("Europe/London"); + Cron cron = new Cron("0 30 1 * * ?", london); // Every day at 01:30 + + Instant beforeTransition = Instant.parse("2024-10-27T00:00:00Z"); + long beforeTransitionEpoch = beforeTransition.toEpochMilli(); + + long firstValidTimeAfter = cron.getNextValidTimeAfter(beforeTransitionEpoch); + assertThat(ofEpochMilli(firstValidTimeAfter), equalTo(Instant.parse("2024-10-27T00:30:00Z"))); + + long nextValidTimeAfter = cron.getNextValidTimeAfter(firstValidTimeAfter); + assertThat(ofEpochMilli(nextValidTimeAfter), equalTo(Instant.parse("2024-10-28T01:30:00Z"))); + } + + // This test checks that once per minute crons will be unaffected by a DST transition + public void testDiscontinuityResolutionForNonHourCronInRandomTimezone() { + var timezone = generateRandomDSTZone(); + + var cron = new Cron("0 * * * * ?", timezone); // Once per minute + + Instant referenceTime = randomInstantBetween(Instant.now(), Instant.now().plus(1826, ChronoUnit.DAYS)); // ~5 years + ZoneOffsetTransition transition1 = timezone.getRules().nextTransition(referenceTime); + + // Currently there are no known timezones with DST transitions shorter than 10 minutes but this guards against future changes + if (Math.abs(transition1.getOffsetBefore().getTotalSeconds() - transition1.getOffsetAfter().getTotalSeconds()) < 600) { + fail("Transition is not long enough to test"); + } + + testNonHourCronTransition(transition1, cron); + + var transition2 = timezone.getRules().nextTransition(transition1.getInstant().plus(1, ChronoUnit.DAYS)); + + testNonHourCronTransition(transition2, cron); + + } + + private static void testNonHourCronTransition(ZoneOffsetTransition transition, Cron cron) { + Instant insideTransition; + if (transition.isGap()) { + insideTransition = transition.getInstant().plus(10, ChronoUnit.MINUTES); + Instant nextTrigger = ofEpochMilli(cron.getNextValidTimeAfter(insideTransition.toEpochMilli())); + assertThat(nextTrigger, equalTo(insideTransition.plus(1, ChronoUnit.MINUTES))); + } else { + insideTransition = transition.getInstant().minus(10, ChronoUnit.MINUTES); + Instant nextTrigger = ofEpochMilli(cron.getNextValidTimeAfter(insideTransition.toEpochMilli())); + assertThat(nextTrigger, equalTo(insideTransition.plus(1, ChronoUnit.MINUTES))); + + insideTransition = insideTransition.plus(transition.getDuration()); + nextTrigger = ofEpochMilli(cron.getNextValidTimeAfter(insideTransition.toEpochMilli())); + assertThat(nextTrigger, equalTo(insideTransition.plus(1, ChronoUnit.MINUTES))); + } + } + + // This test checks that once per day crons will behave correctly during a DST transition + public void testDiscontinuityResolutionForCronInRandomTimezone() { + var timezone = generateRandomDSTZone(); + + Instant referenceTime = randomInstantBetween(Instant.now(), Instant.now().plus(1826, ChronoUnit.DAYS)); // ~5 years + ZoneOffsetTransition transition1 = timezone.getRules().nextTransition(referenceTime); + + // Currently there are no known timezones with DST transitions shorter than 10 minutes but this guards against future changes + if (Math.abs(transition1.getOffsetBefore().getTotalSeconds() - transition1.getOffsetAfter().getTotalSeconds()) < 600) { + fail("Transition is not long enough to test"); + } + + testHourCronTransition(transition1, timezone); + + var transition2 = timezone.getRules().nextTransition(transition1.getInstant().plus(1, ChronoUnit.DAYS)); + + testHourCronTransition(transition2, timezone); + } + + private static void testHourCronTransition(ZoneOffsetTransition transition, ZoneId timezone) { + if (transition.isGap()) { + LocalDateTime targetTime = transition.getDateTimeBefore().plusMinutes(10); + + var cron = new Cron("0 " + targetTime.getMinute() + " " + targetTime.getHour() + " * * ?", timezone); + + long nextTrigger = cron.getNextValidTimeAfter(transition.getInstant().minus(10, ChronoUnit.MINUTES).toEpochMilli()); + + assertThat(ofEpochMilli(nextTrigger), equalTo(transition.getInstant().plus(10, ChronoUnit.MINUTES))); + } else { + LocalDateTime targetTime = transition.getDateTimeAfter().plusMinutes(10); + var cron = new Cron("0 " + targetTime.getMinute() + " " + targetTime.getHour() + " * * ?", timezone); + + long transitionLength = Math.abs(transition.getDuration().toSeconds()); + long firstTrigger = cron.getNextValidTimeAfter( + transition.getInstant().minusSeconds(transitionLength).minus(10, ChronoUnit.MINUTES).toEpochMilli() + ); + + assertThat( + ofEpochMilli(firstTrigger), + equalTo(transition.getInstant().minusSeconds(transitionLength).plus(10, ChronoUnit.MINUTES)) + ); + + var repeatTrigger = cron.getNextValidTimeAfter(firstTrigger + (1000 * 60L)); // 1 minute + + assertThat(repeatTrigger - firstTrigger, Matchers.greaterThan(24 * 60 * 60 * 1000L)); // 24 hours + } + } + +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json index f90d2202db0d3..c7424571dd678 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json @@ -15,14 +15,16 @@ "container.name", "process.thread.name" ] + }, + "mapping": { + "source": { + "mode": "synthetic" + } } }, "codec": "best_compression" }, "mappings": { - "_source": { - "mode": "synthetic" - }, "_meta": { "index-template-version": ${xpack.profiling.template.version}, "index-version": ${xpack.profiling.index.events.version}, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-executables.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-executables.json index f1e5e01d50c16..ac72a03202646 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-executables.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-executables.json @@ -5,13 +5,15 @@ "auto_expand_replicas": "0-1", "refresh_interval": "10s", "hidden": true, - "lifecycle.rollover_alias": "profiling-executables" + "lifecycle.rollover_alias": "profiling-executables", + "mapping": { + "source": { + "mode": "synthetic" + } + } } }, "mappings": { - "_source": { - "mode": "synthetic" - }, "_meta": { "index-template-version": ${xpack.profiling.template.version}, "index-version": ${xpack.profiling.index.executables.version}, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-metrics.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-metrics.json index 35f53a36b2d0b..bb893a07c70a1 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-metrics.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-metrics.json @@ -10,14 +10,16 @@ "@timestamp", "host.id" ] + }, + "mapping": { + "source": { + "mode": "synthetic" + } } }, "codec": "best_compression" }, "mappings": { - "_source": { - "mode": "synthetic" - }, "_meta": { "index-template-version": ${xpack.profiling.template.version}, "index-version": ${xpack.profiling.index.metrics.version}, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-stacktraces.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-stacktraces.json index 6c96fb21673ae..1170e3a32d8e2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-stacktraces.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-stacktraces.json @@ -11,13 +11,15 @@ "field": [ "Stacktrace.frame.ids" ] + }, + "mapping": { + "source": { + "mode": "synthetic" + } } } }, "mappings": { - "_source": { - "mode": "synthetic" - }, "_meta": { "index-template-version": ${xpack.profiling.template.version}, "index-version": ${xpack.profiling.index.stacktraces.version}, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-executables.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-executables.json index 71c4d15989b7a..d5d24a22fc58e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-executables.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-executables.json @@ -7,13 +7,15 @@ "index": { "auto_expand_replicas": "0-1", "refresh_interval": "10s", - "hidden": true + "hidden": true, + "mapping": { + "source": { + "mode": "synthetic" + } + } } }, "mappings": { - "_source": { - "mode": "synthetic" - }, "_meta": { "index-template-version": ${xpack.profiling.template.version}, "index-version": ${xpack.profiling.index.sq.executables.version}, diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-leafframes.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-leafframes.json index 20849bfe8f27d..b56b4b2874743 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-leafframes.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/index-template/profiling-sq-leafframes.json @@ -7,13 +7,15 @@ "index": { "auto_expand_replicas": "0-1", "refresh_interval": "10s", - "hidden": true + "hidden": true, + "mapping": { + "source": { + "mode": "synthetic" + } + } } }, "mappings": { - "_source": { - "mode": "synthetic" - }, "_meta": { "index-template-version": ${xpack.profiling.template.version}, "index-version": ${xpack.profiling.index.sq.leafframes.version}, diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index d13f3cda2a82c..f9b2cc5afe3a5 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -96,7 +96,8 @@ private DeprecationChecks() {} IndexDeprecationChecks::checkIndexDataPath, IndexDeprecationChecks::storeTypeSettingCheck, IndexDeprecationChecks::frozenIndexSettingCheck, - IndexDeprecationChecks::deprecatedCamelCasePattern + IndexDeprecationChecks::deprecatedCamelCasePattern, + IndexDeprecationChecks::checkSourceModeInMapping ); static List> DATA_STREAM_CHECKS = List.of( diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index 8144d960df2e8..aaf58a44a6565 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.engine.frozen.FrozenEngine; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import java.util.ArrayList; @@ -201,6 +202,31 @@ static List findInPropertiesRecursively( return issues; } + static DeprecationIssue checkSourceModeInMapping(IndexMetadata indexMetadata, ClusterState clusterState) { + if (SourceFieldMapper.onOrAfterDeprecateModeVersion(indexMetadata.getCreationVersion())) { + boolean[] useSourceMode = { false }; + fieldLevelMappingIssue(indexMetadata, ((mappingMetadata, sourceAsMap) -> { + Object source = sourceAsMap.get("_source"); + if (source instanceof Map sourceMap) { + if (sourceMap.containsKey("mode")) { + useSourceMode[0] = true; + } + } + })); + if (useSourceMode[0]) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + SourceFieldMapper.DEPRECATION_WARNING, + "https://github.com/elastic/elasticsearch/pull/117172", + SourceFieldMapper.DEPRECATION_WARNING, + false, + null + ); + } + } + return null; + } + static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata, ClusterState clusterState) { List fields = new ArrayList<>(); fieldLevelMappingIssue( diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java index 7badf6926c574..387224408a14b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java @@ -140,7 +140,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public void writeTo(StreamOutput out) throws IOException { out.writeWriteable(configurationOverrides); out.writeBoolean(enabled); - out.writeString(interval.toString()); + out.writeString(interval.expression()); out.writeOptionalInstant(lastSynced); out.writeString(name); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java index 3c08a5ac1e218..008cbca0cd5ea 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java @@ -222,7 +222,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(enabled); - out.writeString(interval.toString()); + out.writeString(interval.expression()); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java index e3eac60703915..a092e17931237 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java @@ -113,7 +113,7 @@ public T[] toArray(T[] a) { @Override public boolean add(Attribute e) { - return delegate.put(e, PRESENT) != null; + return delegate.put(e, PRESENT) == null; } @Override diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index f92c895cc5b7b..02f9752d21e09 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -34,6 +34,7 @@ dependencies { compileOnly project(':modules:lang-painless:spi') compileOnly project(xpackModule('esql-core')) compileOnly project(xpackModule('ml')) + implementation project(xpackModule('kql')) implementation project('compute') implementation project('compute:ann') implementation project(':libs:dissect') @@ -50,6 +51,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('enrich')) testImplementation project(path: xpackModule('spatial')) + testImplementation project(path: xpackModule('kql')) testImplementation project(path: ':modules:reindex') testImplementation project(path: ':modules:parent-join') diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java index 2c36b42dee277..06b890603e489 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java @@ -43,6 +43,7 @@ public abstract class AsyncOperator implements Operator { private final int maxOutstandingRequests; private final LongAdder totalTimeInNanos = new LongAdder(); + private boolean finished = false; private volatile boolean closed = false; diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 265d9f7bd8cd5..2484a428c4b03 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -172,7 +172,8 @@ public final void test() throws Throwable { } protected void shouldSkipTest(String testName) throws IOException { - if (testCase.requiredCapabilities.contains("semantic_text_type")) { + if (testCase.requiredCapabilities.contains("semantic_text_type") + || testCase.requiredCapabilities.contains("semantic_text_aggregations")) { assumeTrue("Inference test service needs to be supported for semantic_text", supportsInferenceTestService()); } checkCapabilities(adminClient(), testFeatureService, testName, testCase); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 0d6659ad37a27..ffbac2829ea4a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -56,6 +56,8 @@ public class CsvTestsDataLoader { private static final TestsDataset APPS = new TestsDataset("apps"); private static final TestsDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short")); private static final TestsDataset LANGUAGES = new TestsDataset("languages"); + private static final TestsDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup") + .withSetting("languages_lookup-settings.json"); private static final TestsDataset ALERTS = new TestsDataset("alerts"); private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs"); private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data"); @@ -93,14 +95,13 @@ public class CsvTestsDataLoader { private static final TestsDataset BOOKS = new TestsDataset("books"); private static final TestsDataset SEMANTIC_TEXT = new TestsDataset("semantic_text").withInferenceEndpoint(true); - private static final String LOOKUP_INDEX_SUFFIX = "_lookup"; - public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), Map.entry(HOSTS.indexName, HOSTS), Map.entry(APPS.indexName, APPS), Map.entry(APPS_SHORT.indexName, APPS_SHORT), Map.entry(LANGUAGES.indexName, LANGUAGES), + Map.entry(LANGUAGES_LOOKUP.indexName, LANGUAGES_LOOKUP), Map.entry(UL_LOGS.indexName, UL_LOGS), Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA), Map.entry(ALERTS.indexName, ALERTS), @@ -130,9 +131,7 @@ public class CsvTestsDataLoader { Map.entry(DISTANCES.indexName, DISTANCES), Map.entry(ADDRESSES.indexName, ADDRESSES), Map.entry(BOOKS.indexName, BOOKS), - Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT), - // JOIN LOOKUP alias - Map.entry(LANGUAGES.indexName + LOOKUP_INDEX_SUFFIX, LANGUAGES.withIndex(LANGUAGES.indexName + LOOKUP_INDEX_SUFFIX)) + Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT) ); private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec new file mode 100644 index 0000000000000..02be58efac774 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -0,0 +1,153 @@ +############################################### +# Tests for KQL function +# + +kqlWithField +required_capability: kql_function + +// tag::kql-with-field[] +FROM books +| WHERE KQL("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +// end::kql-with-field[] + +// tag::kql-with-field-result[] +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; +// end::kql-with-field-result[] + +kqlWithMultipleFields +required_capability: kql_function + +from books +| where kql("title:Return* AND author:*Tolkien") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +kqlWithQueryExpressions +required_capability: kql_function + +from books +| where kql(CONCAT("title:Return*", " AND author:*Tolkien")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +kqlWithConjunction +required_capability: kql_function + +from books +| where kql("title: Rings") and ratings > 4.6 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +kqlWithFunctionPushedToLucene +required_capability: kql_function + +from hosts +| where kql("host: beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +kqlWithNonPushableConjunction +required_capability: kql_function + +from books +| where kql("title: Rings") and length(title) > 75 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +; + +kqlWithMultipleWhereClauses +required_capability: kql_function + +from books +| where kql("title: rings") +| where kql("year > 1 AND year < 2005") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + + +kqlWithMultivaluedTextField +required_capability: kql_function + +from employees +| where kql("job_positions: Tech Lead AND job_positions:(Reporting Analyst)") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10004 | Chirstian | Koblick +10010 | Duangkaew | Piveteau +10011 | Mary | Sluis +10088 | Jungsoon | Syrzycki +10093 | Sailaja | Desikan +10097 | Remzi | Waschkowski +; + +kqlWithMultivaluedNumericField +required_capability: kql_function + +from employees +| where kql("salary_change > 14") +| keep emp_no, first_name, last_name, salary_change; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double +10003 | Parto | Bamford | [12.82, 14.68] +10015 | Guoxiang | Nooteboom | [12.4, 14.25] +10023 | Bojan | Montemayor | [0.8, 14.63] +10040 | Weiyi | Meriste | [-8.94, 1.92, 6.97, 14.74] +10061 | Tse | Herber | [-2.58, -0.95, 14.39] +10065 | Satosi | Awdeh | [-9.81, -1.47, 14.44] +10099 | Valter | Sullins | [-8.78, -3.98, 10.71, 14.26] +; + +testMultiValuedFieldWithConjunction +required_capability: kql_function + +from employees +| where (kql("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F" +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10023 | Bojan | Montemayor +10041 | Uri | Lenart +10044 | Mingsen | Casley +10053 | Sanjiv | Zschoche +10069 | Margareta | Bierman +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json new file mode 100644 index 0000000000000..b73d1f9accf92 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json @@ -0,0 +1,5 @@ +{ + "index": { + "mode": "lookup" + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json index c587b69828170..db15133f036bb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json @@ -72,6 +72,10 @@ "st_base64": { "type": "semantic_text", "inference_id": "test_sparse_inference" + }, + "st_logs": { + "type": "semantic_text", + "inference_id": "test_sparse_inference" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 3e92e55928d64..6039dc05b6c44 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -101,8 +101,8 @@ book_no:keyword | title:text ; -matchMultivaluedTextField -required_capability: match_function +qstrWithMultivaluedTextField +required_capability: qstr_function from employees | where qstr("job_positions: (Tech Lead) AND job_positions:(Reporting Analyst)") @@ -118,8 +118,8 @@ emp_no:integer | first_name:keyword | last_name:keyword 10097 | Remzi | Waschkowski ; -matchMultivaluedNumericField -required_capability: match_function +qstrWithMultivaluedNumericField +required_capability: qstr_function from employees | where qstr("salary_change: [14 TO *]") @@ -137,7 +137,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double ; testMultiValuedFieldWithConjunction -required_capability: match_function +required_capability: qstr_function from employees | where (qstr("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv index 6cae82cfefa0a..bd5fe7fad3a4e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv @@ -1,4 +1,4 @@ -_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text -1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw== -2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8= -3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003, +_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text,st_logs:semantic_text +1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==,"2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553" +2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=,"2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42" +3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,,"2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec index de2a79df06a50..43dc6e4d4acd2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec @@ -88,19 +88,75 @@ _id:keyword | my_field:semantic_text 3 | be excellent to each other ; -simpleStats -required_capability: semantic_text_type +statsWithCount +required_capability: semantic_text_aggregations + +FROM semantic_text METADATA _id +| STATS result = COUNT(st_version) +; + +result:long +2 +; + +statsWithCountDistinct +required_capability: semantic_text_aggregations + +FROM semantic_text METADATA _id +| STATS result = COUNT_DISTINCT(st_version) +; + +result:long +2 +; + +statsWithValues +required_capability: semantic_text_aggregations + +FROM semantic_text METADATA _id +| STATS result = VALUES(st_version) +| EVAL result = MV_SORT(result) +; + +result:keyword +["1.2.3", "9.0.0"] +; + +statsWithMin +required_capability: semantic_text_aggregations + +FROM semantic_text METADATA _id +| STATS result = min(st_version) +; + +result:keyword +1.2.3 +; + +statsWithMax +required_capability: semantic_text_aggregations FROM semantic_text METADATA _id -| STATS COUNT(*) +| STATS result = max(st_version) ; -COUNT(*):long -3 +result:keyword +9.0.0 +; + +statsWithTop +required_capability: semantic_text_aggregations + +FROM semantic_text METADATA _id +| STATS result = top(st_version, 2, "asc") +; + +result:keyword +["1.2.3", "9.0.0"] ; statsWithGrouping -required_capability: semantic_text_type +required_capability: semantic_text_aggregations FROM semantic_text METADATA _id | STATS COUNT(*) BY st_version @@ -132,6 +188,36 @@ COUNT(*):long | my_field:semantic_text 1 | bye bye! ; +grok +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| GROK st_logs """%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num}""" +| KEEP st_logs, date, ip, email, num +| SORT st_logs +; + +st_logs:semantic_text | date:keyword | ip:keyword | email:keyword | num:keyword +2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42 | 2023-01-23T12:15:00.000Z | 127.0.0.1 | some.email@foo.com | 42 +2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42 | 2024-01-23T12:15:00.000Z | 1.2.3.4 | foo@example.com | 42 +2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553 | 2024-12-23T12:15:00.000Z | 1.2.3.4 | example@example.com | 4553 +; + +dissect +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| DISSECT st_logs """%{date} %{ip} %{email} %{num}""" +| KEEP st_logs, date, ip, email, num +| SORT st_logs +; + +st_logs:semantic_text | date:keyword | ip:keyword | email:keyword | num:keyword +2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42 | 2023-01-23T12:15:00.000Z | 127.0.0.1 | some.email@foo.com | 42 +2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42 | 2024-01-23T12:15:00.000Z | 1.2.3.4 | foo@example.com | 42 +2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553 | 2024-12-23T12:15:00.000Z | 1.2.3.4 | example@example.com | 4553 +; + simpleWithLongValue required_capability: semantic_text_type diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index 460ab0f5b8b38..56453a291ea81 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -392,12 +392,13 @@ protected void doRun() throws Exception { .get(); ensureYellowAndNoInitializingShards("test"); request.query("FROM test | LIMIT 10"); - request.pragmas(randomPragmas()); + QueryPragmas pragmas = randomPragmas(); + request.pragmas(pragmas); PlainActionFuture future = new PlainActionFuture<>(); client.execute(EsqlQueryAction.INSTANCE, request, future); ExchangeService exchangeService = internalCluster().getInstance(ExchangeService.class, dataNode); - boolean waitedForPages; - final String sessionId; + final boolean waitedForPages; + final String exchangeId; try { List foundTasks = new ArrayList<>(); assertBusy(() -> { @@ -411,13 +412,22 @@ protected void doRun() throws Exception { assertThat(tasks, hasSize(1)); foundTasks.addAll(tasks); }); - sessionId = foundTasks.get(0).taskId().toString(); + final String sessionId = foundTasks.get(0).taskId().toString(); assertTrue(fetchingStarted.await(1, TimeUnit.MINUTES)); - String exchangeId = exchangeService.sinkKeys().stream().filter(s -> s.startsWith(sessionId)).findFirst().get(); + List sinkKeys = exchangeService.sinkKeys() + .stream() + .filter( + s -> s.startsWith(sessionId) + // exclude the node-level reduction sink + && s.endsWith("[n]") == false + ) + .toList(); + assertThat(sinkKeys.toString(), sinkKeys.size(), equalTo(1)); + exchangeId = sinkKeys.get(0); ExchangeSinkHandler exchangeSink = exchangeService.getSinkHandler(exchangeId); waitedForPages = randomBoolean(); if (waitedForPages) { - // do not fail exchange requests until we have some pages + // do not fail exchange requests until we have some pages. assertBusy(() -> assertThat(exchangeSink.bufferSize(), greaterThan(0))); } } finally { @@ -429,7 +439,7 @@ protected void doRun() throws Exception { // As a result, the exchange sinks on data-nodes won't be removed until the inactive_timeout elapses, which is // longer than the assertBusy timeout. if (waitedForPages == false) { - exchangeService.finishSinkHandler(sessionId, failure); + exchangeService.finishSinkHandler(exchangeId, failure); } } finally { transportService.clearAllRules(); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java new file mode 100644 index 0000000000000..d58637ab52c86 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.CoreMatchers.containsString; + +public class KqlFunctionIT extends AbstractEsqlIntegTestCase { + + @BeforeClass + protected static void ensureKqlFunctionEnabled() { + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + } + + @Before + public void setupIndex() { + createAndPopulateIndex(); + } + + public void testSimpleKqlQuery() { + var query = """ + FROM test + | WHERE kql("content: dog") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(1), List.of(3), List.of(4), List.of(5))); + } + } + + public void testMultiFieldKqlQuery() { + var query = """ + FROM test + | WHERE kql("dog OR canine") + | KEEP id + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValuesInAnyOrder(resp.values(), List.of(List.of(1), List.of(2), List.of(3), List.of(4), List.of(5))); + } + } + + public void testKqlQueryWithinEval() { + var query = """ + FROM test + | EVAL matches_query = kql("title: fox") + """; + + var error = expectThrows(VerificationException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands")); + } + + public void testInvalidKqlQueryEof() { + var query = """ + FROM test + | WHERE kql("content: ((((dog") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse KQL query [content: ((((dog]")); + assertThat(error.getRootCause().getMessage(), containsString("line 1:11: mismatched input '('")); + } + + public void testInvalidKqlQueryLexicalError() { + var query = """ + FROM test + | WHERE kql(":") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse KQL query [:]")); + assertThat(error.getRootCause().getMessage(), containsString("line 1:1: extraneous input ':' ")); + } + + private void createAndPopulateIndex() { + var indexName = "test"; + var client = client().admin().indices(); + var CreateRequest = client.prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", 1)) + .setMapping("id", "type=integer", "content", "type=text"); + assertAcked(CreateRequest); + client().prepareBulk() + .add( + new IndexRequest(indexName).id("1") + .source("id", 1, "content", "The quick brown animal swiftly jumps over a lazy dog", "title", "A Swift Fox's Journey") + ) + .add( + new IndexRequest(indexName).id("2") + .source("id", 2, "content", "A speedy brown fox hops effortlessly over a sluggish canine", "title", "The Fox's Leap") + ) + .add( + new IndexRequest(indexName).id("3") + .source("id", 3, "content", "Quick and nimble, the fox vaults over the lazy dog", "title", "Brown Fox in Action") + ) + .add( + new IndexRequest(indexName).id("4") + .source( + "id", + 4, + "content", + "A fox that is quick and brown jumps over a dog that is quite lazy", + "title", + "Speedy Animals" + ) + ) + .add( + new IndexRequest(indexName).id("5") + .source( + "id", + 5, + "content", + "With agility, a quick brown fox bounds over a slow-moving dog", + "title", + "Foxes and Canines" + ) + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + ensureYellow(indexName); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d675f772b5a3b..08fa7f0a9b213 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -415,6 +415,11 @@ public enum Cap { */ MATCH_FUNCTION, + /** + * KQL function + */ + KQL_FUNCTION(Build.current().isSnapshot()), + /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. * https://github.com/elastic/elasticsearch/issues/112704 @@ -521,7 +526,12 @@ public enum Cap { /** * Fix for https://github.com/elastic/elasticsearch/issues/117054 */ - FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER; + FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER, + + /** + * support for aggregations on semantic_text + */ + SEMANTIC_TEXT_AGGREGATIONS(EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 7ad4c3d3e644d..dde7bc09ac615 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -62,6 +62,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.TableIdentifier; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; @@ -106,7 +107,6 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -199,11 +199,12 @@ private static class ResolveTable extends ParameterizedAnalyzerRule"), enrichResolution); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 3ebb52641232e..2be13398dab2f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; @@ -793,14 +794,19 @@ private static void checkNotPresentInDisjunctions( private static void checkFullTextQueryFunctions(LogicalPlan plan, Set failures) { if (plan instanceof Filter f) { Expression condition = f.condition(); - checkCommandsBeforeExpression( - plan, - condition, - QueryString.class, - lp -> (lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation), - qsf -> "[" + qsf.functionName() + "] " + qsf.functionType(), - failures - ); + + List.of(QueryString.class, Kql.class).forEach(functionClass -> { + // Check for limitations of QSTR and KQL function. + checkCommandsBeforeExpression( + plan, + condition, + functionClass, + lp -> (lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation), + fullTextFunction -> "[" + fullTextFunction.functionName() + "] " + fullTextFunction.functionType(), + failures + ); + }); + checkCommandsBeforeExpression( plan, condition, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index ea1669ccc7a4f..3d26bc170b723 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; @@ -411,6 +412,7 @@ private static FunctionDefinition[][] snapshotFunctions() { // This is an experimental function and can be removed without notice. def(Delay.class, Delay::new, "delay"), def(Categorize.class, Categorize::new, "categorize"), + def(Kql.class, Kql::new, "kql"), def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 5ae162f1fbb12..2e45b1c1fe082 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -66,7 +66,8 @@ public class CountDistinct extends AggregateFunction implements OptionalArgument Map.entry(DataType.KEYWORD, CountDistinctBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.IP, CountDistinctBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, CountDistinctBytesRefAggregatorFunctionSupplier::new), - Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new) + Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.SEMANTIC_TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new) ); private static final int DEFAULT_PRECISION = 3000; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 2165c3c7ad1a0..eb0c8abd1080b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -51,6 +51,7 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp Map.entry(DataType.IP, MaxIpAggregatorFunctionSupplier::new), Map.entry(DataType.KEYWORD, MaxBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.TEXT, MaxBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.SEMANTIC_TEXT, MaxBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, MaxBytesRefAggregatorFunctionSupplier::new) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index 7d67868dd4134..472f0b1ff5cd1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -51,7 +51,8 @@ public class Min extends AggregateFunction implements ToAggregator, SurrogateExp Map.entry(DataType.IP, MinIpAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, MinBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.KEYWORD, MinBytesRefAggregatorFunctionSupplier::new), - Map.entry(DataType.TEXT, MinBytesRefAggregatorFunctionSupplier::new) + Map.entry(DataType.TEXT, MinBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.SEMANTIC_TEXT, MinBytesRefAggregatorFunctionSupplier::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java index e7df990b20422..5260b3e8fa279 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java @@ -46,6 +46,7 @@ public class Values extends AggregateFunction implements ToAggregator { Map.entry(DataType.DOUBLE, ValuesDoubleAggregatorFunctionSupplier::new), Map.entry(DataType.KEYWORD, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.TEXT, ValuesBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.SEMANTIC_TEXT, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.IP, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, ValuesBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.BOOLEAN, ValuesBooleanAggregatorFunctionSupplier::new) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java index d59c736783172..8804a031de78c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextWritables.java @@ -8,14 +8,28 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class FullTextWritables { public static List getNamedWriteables() { - return List.of(MatchQueryPredicate.ENTRY, MultiMatchQueryPredicate.ENTRY, QueryString.ENTRY, Match.ENTRY); + List entries = new ArrayList<>(); + + entries.add(MatchQueryPredicate.ENTRY); + entries.add(MultiMatchQueryPredicate.ENTRY); + entries.add(QueryString.ENTRY); + entries.add(Match.ENTRY); + + if (EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()) { + entries.add(Kql.ENTRY); + } + + return Collections.unmodifiableList(entries); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java new file mode 100644 index 0000000000000..c03902373c02e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; + +import java.io.IOException; +import java.util.List; + +/** + * Full text function that performs a {@link KqlQuery} . + */ +public class Kql extends FullTextFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::new); + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a KQL query. Returns true if the provided KQL query string matches the row.", + examples = { @Example(file = "kql-function", tag = "kql-with-field") } + ) + public Kql( + Source source, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Query string in KQL query string format." + ) Expression queryString + ) { + super(source, queryString, List.of(queryString)); + } + + private Kql(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Kql(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Kql::new, query()); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 9f574ee8005b2..3d6c35e914294 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -30,8 +30,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; -import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; @@ -252,10 +252,10 @@ static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePu && Expressions.foldable(cidrMatch.matches()); } else if (exp instanceof SpatialRelatesFunction spatial) { return canPushSpatialFunctionToSource(spatial, lucenePushdownPredicates); - } else if (exp instanceof QueryString) { - return true; } else if (exp instanceof Match mf) { return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType()); + } else if (exp instanceof FullTextFunction) { + return true; } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 605e0d7c3109c..18bbfdf485a81 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -302,12 +302,13 @@ private static String dataTypeToString(DataType type, Class aggClass) { case DataType.INTEGER, DataType.COUNTER_INTEGER -> "Int"; case DataType.LONG, DataType.DATETIME, DataType.COUNTER_LONG, DataType.DATE_NANOS -> "Long"; case DataType.DOUBLE, DataType.COUNTER_DOUBLE -> "Double"; - case DataType.KEYWORD, DataType.IP, DataType.VERSION, DataType.TEXT -> "BytesRef"; + case DataType.KEYWORD, DataType.IP, DataType.VERSION, DataType.TEXT, DataType.SEMANTIC_TEXT -> "BytesRef"; case GEO_POINT -> "GeoPoint"; case CARTESIAN_POINT -> "CartesianPoint"; - case SEMANTIC_TEXT, UNSUPPORTED, NULL, UNSIGNED_LONG, SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, SOURCE, DATE_PERIOD, - TIME_DURATION, CARTESIAN_SHAPE, GEO_SHAPE, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> - throw new EsqlIllegalArgumentException("illegal agg type: " + type.typeName()); + case UNSUPPORTED, NULL, UNSIGNED_LONG, SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, SOURCE, DATE_PERIOD, TIME_DURATION, + CARTESIAN_SHAPE, GEO_SHAPE, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new EsqlIllegalArgumentException( + "illegal agg type: " + type.typeName() + ); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 6fac7bab2bd80..1580b77931240 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; @@ -47,6 +48,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; import org.elasticsearch.xpack.esql.querydsl.query.SpatialRelatesQuery; import org.elasticsearch.xpack.versionfield.Version; @@ -89,6 +91,7 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.MultiMatches(), new MatchFunctionTranslator(), new QueryStringFunctionTranslator(), + new KqlFunctionTranslator(), new Scalars() ); @@ -538,4 +541,11 @@ protected Query asQuery(QueryString queryString, TranslatorHandler handler) { return new QueryStringQuery(queryString.source(), queryString.queryAsText(), Map.of(), Map.of()); } } + + public static class KqlFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(Kql kqlFunction, TranslatorHandler handler) { + return new KqlQuery(kqlFunction.source(), kqlFunction.queryAsText()); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java new file mode 100644 index 0000000000000..c388a131b9ab6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.querydsl.query; + +import org.elasticsearch.core.Booleans; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiConsumer; + +import static java.util.Map.entry; + +public class KqlQuery extends Query { + + private static final Map> BUILDER_APPLIERS = Map.ofEntries( + entry(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), KqlQueryBuilder::timeZone), + entry(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), KqlQueryBuilder::defaultField), + entry(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), (qb, s) -> qb.caseInsensitive(Booleans.parseBoolean(s))) + ); + + private final String query; + + private final Map options; + + // dedicated constructor for QueryTranslator + public KqlQuery(Source source, String query) { + this(source, query, null); + } + + public KqlQuery(Source source, String query, Map options) { + super(source); + this.query = query; + this.options = options == null ? Collections.emptyMap() : options; + } + + @Override + public QueryBuilder asBuilder() { + final KqlQueryBuilder queryBuilder = new KqlQueryBuilder(query); + options.forEach((k, v) -> { + if (BUILDER_APPLIERS.containsKey(k)) { + BUILDER_APPLIERS.get(k).accept(queryBuilder, v); + } else { + throw new IllegalArgumentException("illegal kql query option [" + k + "]"); + } + }); + return queryBuilder; + } + + public String query() { + return query; + } + + public Map options() { + return options; + } + + @Override + public int hashCode() { + return Objects.hash(query, options); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + + KqlQuery other = (KqlQuery) obj; + return Objects.equals(query, other.query) && Objects.equals(options, other.options); + } + + @Override + protected String innerToString() { + return query; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 9630a520e8654..25bb6d80d0dd0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.compute.data.Block; @@ -62,6 +63,8 @@ import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; +import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; @@ -76,7 +79,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -272,9 +274,12 @@ public void analyzedPlan(LogicalPlan parsed, EsqlExecutionInfo executionInfo, Ac return; } - preAnalyze(parsed, executionInfo, (indices, policies) -> { + preAnalyze(parsed, executionInfo, (indices, lookupIndices, policies) -> { planningMetrics.gatherPreAnalysisMetrics(parsed); - Analyzer analyzer = new Analyzer(new AnalyzerContext(configuration, functionRegistry, indices, policies), verifier); + Analyzer analyzer = new Analyzer( + new AnalyzerContext(configuration, functionRegistry, indices, lookupIndices, policies), + verifier + ); var plan = analyzer.analyze(parsed); plan.setAnalyzed(); LOGGER.debug("Analyzed plan:\n{}", plan); @@ -285,7 +290,7 @@ public void analyzedPlan(LogicalPlan parsed, EsqlExecutionInfo executionInfo, Ac private void preAnalyze( LogicalPlan parsed, EsqlExecutionInfo executionInfo, - BiFunction action, + TriFunction action, ActionListener listener ) { PreAnalyzer.PreAnalysis preAnalysis = preAnalyzer.preAnalyze(parsed); @@ -299,63 +304,81 @@ private void preAnalyze( ).keySet(); enrichPolicyResolver.resolvePolicies(targetClusters, unresolvedPolicies, listener.delegateFailureAndWrap((l, enrichResolution) -> { // first we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API - var matchFields = enrichResolution.resolvedEnrichPolicies() + var enrichMatchFields = enrichResolution.resolvedEnrichPolicies() .stream() .map(ResolvedEnrichPolicy::matchField) .collect(Collectors.toSet()); - Map unavailableClusters = enrichResolution.getUnavailableClusters(); - preAnalyzeIndices(parsed, executionInfo, unavailableClusters, l.delegateFailureAndWrap((ll, indexResolution) -> { - // TODO in follow-PR (for skip_unavailble handling of missing concrete indexes) add some tests for invalid index - // resolution to updateExecutionInfo - if (indexResolution.isValid()) { - EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); - EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.unavailableClusters()); - if (executionInfo.isCrossClusterSearch() - && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) { - // for a CCS, if all clusters have been marked as SKIPPED, nothing to search so send a sentinel - // Exception to let the LogicalPlanActionListener decide how to proceed - ll.onFailure(new NoClustersToSearchException()); - return; - } - - Set newClusters = enrichPolicyResolver.groupIndicesPerCluster( - indexResolution.get().concreteIndices().toArray(String[]::new) - ).keySet(); - // If new clusters appear when resolving the main indices, we need to resolve the enrich policies again - // or exclude main concrete indices. Since this is rare, it's simpler to resolve the enrich policies again. - // TODO: add a test for this - if (targetClusters.containsAll(newClusters) == false - // do not bother with a re-resolution if only remotes were requested and all were offline - && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) > 0) { - enrichPolicyResolver.resolvePolicies( - newClusters, - unresolvedPolicies, - ll.map(newEnrichResolution -> action.apply(indexResolution, newEnrichResolution)) - ); - return; - } - } - ll.onResponse(action.apply(indexResolution, enrichResolution)); - }), matchFields); + // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy + var fieldNames = fieldNames(parsed, enrichMatchFields); + // First resolve the lookup indices, then the main indices + preAnalyzeLookupIndices( + preAnalysis.lookupIndices, + fieldNames, + l.delegateFailureAndWrap( + (lx, lookupIndexResolution) -> preAnalyzeIndices( + indices, + executionInfo, + enrichResolution.getUnavailableClusters(), + fieldNames, + lx.delegateFailureAndWrap((ll, indexResolution) -> { + // TODO in follow-PR (for skip_unavailble handling of missing concrete indexes) add some tests for invalid + // index resolution to updateExecutionInfo + if (indexResolution.isValid()) { + EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); + EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters( + executionInfo, + indexResolution.unavailableClusters() + ); + if (executionInfo.isCrossClusterSearch() + && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) { + // for a CCS, if all clusters have been marked as SKIPPED, nothing to search so send a sentinel + // Exception to let the LogicalPlanActionListener decide how to proceed + ll.onFailure(new NoClustersToSearchException()); + return; + } + + Set newClusters = enrichPolicyResolver.groupIndicesPerCluster( + indexResolution.get().concreteIndices().toArray(String[]::new) + ).keySet(); + // If new clusters appear when resolving the main indices, we need to resolve the enrich policies again + // or exclude main concrete indices. Since this is rare, it's simpler to resolve the enrich policies + // again. + // TODO: add a test for this + if (targetClusters.containsAll(newClusters) == false + // do not bother with a re-resolution if only remotes were requested and all were offline + && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) > 0) { + enrichPolicyResolver.resolvePolicies( + newClusters, + unresolvedPolicies, + ll.map( + newEnrichResolution -> action.apply(indexResolution, lookupIndexResolution, newEnrichResolution) + ) + ); + return; + } + } + ll.onResponse(action.apply(indexResolution, lookupIndexResolution, enrichResolution)); + }) + ) + ) + ); })); } private void preAnalyzeIndices( - LogicalPlan parsed, + List indices, EsqlExecutionInfo executionInfo, Map unavailableClusters, // known to be unavailable from the enrich policy API call - ActionListener listener, - Set enrichPolicyMatchFields + Set fieldNames, + ActionListener listener ) { - PreAnalyzer.PreAnalysis preAnalysis = new PreAnalyzer().preAnalyze(parsed); // TODO we plan to support joins in the future when possible, but for now we'll just fail early if we see one - if (preAnalysis.indices.size() > 1) { + if (indices.size() > 1) { // Note: JOINs are not supported but we detect them when listener.onFailure(new MappingException("Queries with multiple indices are not supported")); - } else if (preAnalysis.indices.size() == 1) { - TableInfo tableInfo = preAnalysis.indices.get(0); + } else if (indices.size() == 1) { + TableInfo tableInfo = indices.get(0); TableIdentifier table = tableInfo.id(); - var fieldNames = fieldNames(parsed, enrichPolicyMatchFields); Map clusterIndices = indicesExpressionGrouper.groupIndices(IndicesOptions.DEFAULT, table.index()); for (Map.Entry entry : clusterIndices.entrySet()) { @@ -401,6 +424,25 @@ private void preAnalyzeIndices( } } + private void preAnalyzeLookupIndices(List indices, Set fieldNames, ActionListener listener) { + if (indices.size() > 1) { + // Note: JOINs on more than one index are not yet supported + listener.onFailure(new MappingException("More than one LOOKUP JOIN is not supported")); + } else if (indices.size() == 1) { + TableInfo tableInfo = indices.get(0); + TableIdentifier table = tableInfo.id(); + // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types + indexResolver.resolveAsMergedMapping(table.index(), fieldNames, listener); + } else { + try { + // No lookup indices specified + listener.onResponse(IndexResolution.invalid("[none specified]")); + } catch (Exception ex) { + listener.onFailure(ex); + } + } + } + static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchFields) { if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) { // no explicit columns selection, for example "from employees" @@ -422,6 +464,7 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF // "keep" attributes are special whenever a wildcard is used in their name // ie "from test | eval lang = languages + 1 | keep *l" should consider both "languages" and "*l" as valid fields to ask for AttributeSet keepCommandReferences = new AttributeSet(); + AttributeSet keepJoinReferences = new AttributeSet(); List> keepMatches = new ArrayList<>(); List keepPatterns = new ArrayList<>(); @@ -440,6 +483,11 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF // The exact name of the field will be added later as part of enrichPolicyMatchFields Set enrichRefs.removeIf(attr -> attr instanceof EmptyAttribute); references.addAll(enrichRefs); + } else if (p instanceof LookupJoin join) { + keepJoinReferences.addAll(join.config().matchFields()); // TODO: why is this empty + if (join.config().type() instanceof JoinTypes.UsingJoinType usingJoinType) { + keepJoinReferences.addAll(usingJoinType.columns()); + } } else { references.addAll(p.references()); if (p instanceof UnresolvedRelation ur && ur.indexMode() == IndexMode.TIME_SERIES) { @@ -473,6 +521,8 @@ static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchF references.removeIf(attr -> matchByName(attr, alias.name(), keepCommandReferences.contains(attr))); }); }); + // Add JOIN ON column references afterward to avoid Alias removal + references.addAll(keepJoinReferences); // remove valid metadata attributes because they will be filtered out by the IndexResolver anyway // otherwise, in some edge cases, we will fail to ask for "*" (all fields) instead diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 012720db9efd9..010a60ef7da15 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -257,6 +257,10 @@ public final void test() throws Throwable { "can't use MATCH function in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_FUNCTION.capabilityName()) ); + assumeFalse( + "can't use KQL function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.KQL_FUNCTION.capabilityName()) + ); assumeFalse( "lookup join disabled for csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP.capabilityName()) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 7b2f85b80b3b6..f25b19c4e5d1c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1263,11 +1263,74 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { ); } + public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + // Source commands + assertEquals("1:13: [KQL] function cannot be used after SHOW", error("show info | where kql(\"8.16.0\")")); + assertEquals("1:17: [KQL] function cannot be used after ROW", error("row a= \"Anna\" | where kql(\"Anna\")")); + + // Processing commands + assertEquals( + "1:43: [KQL] function cannot be used after DISSECT", + error("from test | dissect first_name \"%{foo}\" | where kql(\"Connection\")") + ); + assertEquals("1:27: [KQL] function cannot be used after DROP", error("from test | drop emp_no | where kql(\"Anna\")")); + assertEquals( + "1:71: [KQL] function cannot be used after ENRICH", + error("from test | enrich languages on languages with lang = language_name | where kql(\"Anna\")") + ); + assertEquals("1:26: [KQL] function cannot be used after EVAL", error("from test | eval z = 2 | where kql(\"Anna\")")); + assertEquals( + "1:44: [KQL] function cannot be used after GROK", + error("from test | grok last_name \"%{WORD:foo}\" | where kql(\"Anna\")") + ); + assertEquals("1:27: [KQL] function cannot be used after KEEP", error("from test | keep emp_no | where kql(\"Anna\")")); + assertEquals("1:24: [KQL] function cannot be used after LIMIT", error("from test | limit 10 | where kql(\"Anna\")")); + assertEquals("1:35: [KQL] function cannot be used after MV_EXPAND", error("from test | mv_expand last_name | where kql(\"Anna\")")); + assertEquals( + "1:45: [KQL] function cannot be used after RENAME", + error("from test | rename last_name as full_name | where kql(\"Anna\")") + ); + assertEquals( + "1:52: [KQL] function cannot be used after STATS", + error("from test | STATS c = COUNT(emp_no) BY languages | where kql(\"Anna\")") + ); + + // Some combination of processing commands + assertEquals("1:38: [KQL] function cannot be used after LIMIT", error("from test | keep emp_no | limit 10 | where kql(\"Anna\")")); + assertEquals( + "1:46: [KQL] function cannot be used after MV_EXPAND", + error("from test | limit 10 | mv_expand last_name | where kql(\"Anna\")") + ); + assertEquals( + "1:52: [KQL] function cannot be used after KEEP", + error("from test | mv_expand last_name | keep last_name | where kql(\"Anna\")") + ); + assertEquals( + "1:77: [KQL] function cannot be used after RENAME", + error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where kql(\"Anna\")") + ); + assertEquals( + "1:54: [KQL] function cannot be used after DROP", + error("from test | rename last_name as name | drop emp_no | where kql(\"Anna\")") + ); + } + public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); } + public void testKqlFunctionOnlyAllowedInWhere() throws Exception { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")")); + checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); + } + public void testMatchFunctionOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")", "function"); } @@ -1309,10 +1372,29 @@ public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { // Other value types are tested in QueryStringFunctionTests } + public void testKqlFunctionArgNotNullOrConstant() throws Exception { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + assertEquals( + "1:19: argument of [kql(first_name)] must be a constant, received [first_name]", + error("from test | where kql(first_name)") + ); + assertEquals("1:19: argument of [kql(null)] cannot be null, received [null]", error("from test | where kql(null)")); + // Other value types are tested in KqlFunctionTests + } + public void testQueryStringWithDisjunctions() { checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); } + public void testKqlFunctionWithDisjunctions() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + checkWithDisjunctions("KQL", "kql(\"first_name: Anna\")", "function"); + } + public void testMatchFunctionWithDisjunctions() { checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")", "function"); } @@ -1368,6 +1450,13 @@ public void testQueryStringFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); } + public void testKqlFunctionWithNonBooleanFunctions() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"first_name: Anna\")", "function"); + } + public void testMatchFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")", "function"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java index fff2d824fc710..e0b8c1356d087 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java @@ -57,7 +57,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.SEMANTIC_TEXT) ).flatMap(List::stream).forEach(fieldCaseSupplier -> { // With precision for (var precisionCaseSupplier : precisionSuppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java index 979048534edbf..131072acff870 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java @@ -47,7 +47,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.geoPointCases(1, 1000, true), MultiRowTestCaseSupplier.cartesianPointCases(1, 1000, true), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.SEMANTIC_TEXT) ).flatMap(List::stream).map(CountTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); // No rows diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java index 7d4b46f2a902a..ae5b3691b0a7d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java @@ -48,7 +48,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.SEMANTIC_TEXT) ).flatMap(List::stream).map(MaxTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); suppliers.addAll( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java index 58ef8d86017a8..ad2953f057635 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java @@ -48,7 +48,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.SEMANTIC_TEXT) ).flatMap(List::stream).map(MinTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); suppliers.addAll( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java index f7bf338caa099..f236e4d8faf98 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java @@ -48,7 +48,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.booleanCases(1, 1000), MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.SEMANTIC_TEXT) ) .flatMap(List::stream) .map(fieldCaseSupplier -> TopTests.makeSupplier(fieldCaseSupplier, limitCaseSupplier, order)) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java index 29faceee7497e..5f35f8cada397 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java @@ -51,7 +51,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.versionCases(1, 1000), // Lower values for strings, as they take more space and may trigger the circuit breaker MultiRowTestCaseSupplier.stringCases(1, 20, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 20, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 20, DataType.TEXT), + MultiRowTestCaseSupplier.stringCases(1, 20, DataType.SEMANTIC_TEXT) ).flatMap(List::stream).map(ValuesTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); return parameterSuppliersFromTypedDataWithDefaultChecks( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java new file mode 100644 index 0000000000000..d97be6b169eef --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KqlTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.junit.BeforeClass; + +import java.util.List; +import java.util.function.Supplier; + +public class KqlTests extends NoneFieldFullTextFunctionTestCase { + @BeforeClass + protected static void ensureKqlFunctionEnabled() { + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + } + + public KqlTests(@Name("TestCase") Supplier testCaseSupplier) { + super(testCaseSupplier); + } + + @ParametersFactory + public static Iterable parameters() { + return generateParameters(); + } + + @Override + protected Expression build(Source source, List args) { + return new Kql(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/NoneFieldFullTextFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/NoneFieldFullTextFunctionTestCase.java new file mode 100644 index 0000000000000..383cb8671053d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/NoneFieldFullTextFunctionTestCase.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class NoneFieldFullTextFunctionTestCase extends AbstractFunctionTestCase { + + public NoneFieldFullTextFunctionTestCase(Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + public final void testFold() { + Expression expression = buildLiteralExpression(testCase); + if (testCase.getExpectedTypeError() != null) { + assertTypeResolutionFailure(expression); + return; + } + assertFalse("expected resolved", expression.typeResolved().unresolved()); + } + + protected static Iterable generateParameters() { + List suppliers = new LinkedList<>(); + for (DataType strType : DataType.stringTypes()) { + suppliers.add( + new TestCaseSupplier( + "<" + strType + ">", + List.of(strType), + () -> testCase(strType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) + ) + ); + } + List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); + } + + private static TestCaseSupplier.TestCase testCase(DataType strType, String str, Matcher matcher) { + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(str), strType, "query")), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + matcher + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java index b4b4ebcaacde6..f573e59ab205a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java @@ -10,61 +10,24 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.hamcrest.Matcher; -import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - @FunctionName("qstr") -public class QueryStringTests extends AbstractFunctionTestCase { +public class QueryStringTests extends NoneFieldFullTextFunctionTestCase { public QueryStringTests(@Name("TestCase") Supplier testCaseSupplier) { - this.testCase = testCaseSupplier.get(); + super(testCaseSupplier); } @ParametersFactory public static Iterable parameters() { - List suppliers = new LinkedList<>(); - for (DataType strType : DataType.stringTypes()) { - suppliers.add( - new TestCaseSupplier( - "<" + strType + ">", - List.of(strType), - () -> testCase(strType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) - ) - ); - } - List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); - // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests - return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); - } - - public final void testFold() { - Expression expression = buildLiteralExpression(testCase); - if (testCase.getExpectedTypeError() != null) { - assertTypeResolutionFailure(expression); - return; - } - assertFalse("expected resolved", expression.typeResolved().unresolved()); - } - - private static TestCaseSupplier.TestCase testCase(DataType strType, String str, Matcher matcher) { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(new BytesRef(str), strType, "query")), - "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", - DataType.BOOLEAN, - matcher - ); + return generateParameters(); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 269b4806680a6..4612ccb425ba2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; @@ -62,6 +63,7 @@ import org.elasticsearch.xpack.esql.stats.Metrics; import org.elasticsearch.xpack.esql.stats.SearchContextStats; import org.elasticsearch.xpack.esql.stats.SearchStats; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; import org.junit.Before; import java.io.IOException; @@ -678,7 +680,7 @@ public void testMatchFunctionMultipleWhereClauses() { * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, * {"match":{"first_name":{"query":"John"}}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] */ - public void testMatchFunctionMultipleQstrClauses() { + public void testMatchFunctionMultipleMatchClauses() { String queryText = """ from test | where match(last_name, "Smith") and match(first_name, "John") @@ -698,6 +700,182 @@ public void testMatchFunctionMultipleQstrClauses() { assertThat(query.query().toString(), is(expected.toString())); } + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"kql":{"query":"last_name: Smith"}}] + */ + public void testKqlFunction() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + var plan = plannerOptimizer.plan(""" + from test + | where kql("last_name: Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = kqlQueryBuilder("last_name: Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false] + * \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]] + * \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"kql":{"query":"last_name: Smith"}} + * ,{"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010"}}], + * "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324] + */ + public void testKqlFunctionConjunctionWhereOperands() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") and emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 36, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var kqlQuery = kqlQueryBuilder("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(kqlQuery).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"kql":{"query":"last_name: Smith"}},{ + * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] + */ + public void testKqlFunctionWithFunctionsPushedToLucene() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json"); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 36, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var kqlQuery = kqlQueryBuilder("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(kqlQuery).must(terms); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false] + * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] + * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], + * query[{"bool":{"must":[{"kql":{"query":"last_name: Smith"}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], + * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + */ + public void testKqlFunctionMultipleWhereClauses() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") + | where emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(3, 8, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var kqlQuery = kqlQueryBuilder("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(kqlQuery).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool": {"must":[ + * {"kql":{"query":"last_name: Smith"}}, + * {"kql":{"query":"emp_no > 10010"}}],"boost":1.0}}] + */ + public void testKqlFunctionMultipleKqlClauses() { + // Skip test if the kql function is not enabled. + assumeTrue("kql function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + + String queryText = """ + from test + | where kql("last_name: Smith") and kql("emp_no > 10010") + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + var kqlQueryLeft = kqlQueryBuilder("last_name: Smith"); + var kqlQueryRight = kqlQueryBuilder("emp_no > 10010"); + var expected = QueryBuilders.boolQuery().must(kqlQueryLeft).must(kqlQueryRight); + assertThat(query.query().toString(), is(expected.toString())); + } + // optimizer doesn't know yet how to break down different multi count public void testCountFieldsAndAllWithFilter() { var plan = plannerOptimizer.plan(""" @@ -1166,4 +1344,8 @@ private Stat queryStatsFor(PhysicalPlan plan) { protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); } + + private static KqlQueryBuilder kqlQueryBuilder(String query) { + return new KqlQueryBuilder(query); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java new file mode 100644 index 0000000000000..8dfb50f84ac1e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQueryTests.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.querydsl.query; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.tree.SourceTests; +import org.elasticsearch.xpack.esql.core.util.StringUtils; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; + +import java.time.ZoneId; +import java.time.zone.ZoneRulesException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.hamcrest.Matchers.equalTo; + +public class KqlQueryTests extends ESTestCase { + static KqlQuery randomKqkQueryQuery() { + Map options = new HashMap<>(); + + if (randomBoolean()) { + options.put(KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), String.valueOf(randomBoolean())); + } + + if (randomBoolean()) { + options.put(KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), randomIdentifier()); + } + + if (randomBoolean()) { + options.put(KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), randomZone().getId()); + } + + return new KqlQuery(SourceTests.randomSource(), randomAlphaOfLength(5), Collections.unmodifiableMap(options)); + } + + public void testEqualsAndHashCode() { + for (int runs = 0; runs < 100; runs++) { + checkEqualsAndHashCode(randomKqkQueryQuery(), KqlQueryTests::copy, KqlQueryTests::mutate); + } + } + + private static KqlQuery copy(KqlQuery query) { + return new KqlQuery(query.source(), query.query(), query.options()); + } + + private static KqlQuery mutate(KqlQuery query) { + List> options = Arrays.asList( + q -> new KqlQuery(SourceTests.mutate(q.source()), q.query(), q.options()), + q -> new KqlQuery(q.source(), randomValueOtherThan(q.query(), () -> randomAlphaOfLength(5)), q.options()), + q -> new KqlQuery(q.source(), q.query(), mutateOptions(q.options())) + ); + + return randomFrom(options).apply(query); + } + + private static Map mutateOptions(Map options) { + Map mutatedOptions = new HashMap<>(options); + if (options.isEmpty() == false && randomBoolean()) { + mutatedOptions = options.entrySet() + .stream() + .filter(entry -> randomBoolean()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + while (mutatedOptions.equals(options)) { + if (randomBoolean()) { + mutatedOptions = mutateOption( + mutatedOptions, + KqlQueryBuilder.CASE_INSENSITIVE_FIELD.getPreferredName(), + () -> String.valueOf(randomBoolean()) + ); + } + + if (randomBoolean()) { + mutatedOptions = mutateOption( + mutatedOptions, + KqlQueryBuilder.DEFAULT_FIELD_FIELD.getPreferredName(), + () -> randomIdentifier() + ); + } + + if (randomBoolean()) { + mutatedOptions = mutateOption( + mutatedOptions, + KqlQueryBuilder.TIME_ZONE_FIELD.getPreferredName(), + () -> randomZone().getId() + ); + } + } + + return Collections.unmodifiableMap(mutatedOptions); + } + + private static Map mutateOption(Map options, String optionName, Supplier valueSupplier) { + options = new HashMap<>(options); + options.put(optionName, randomValueOtherThan(options.get(optionName), valueSupplier)); + return options; + } + + public void testQueryBuilding() { + KqlQueryBuilder qb = getBuilder(Map.of("case_insensitive", "false")); + assertThat(qb.caseInsensitive(), equalTo(false)); + + qb = getBuilder(Map.of("case_insensitive", "false", "time_zone", "UTC", "default_field", "foo")); + assertThat(qb.caseInsensitive(), equalTo(false)); + assertThat(qb.timeZone(), equalTo(ZoneId.of("UTC"))); + assertThat(qb.defaultField(), equalTo("foo")); + + Exception e = expectThrows(IllegalArgumentException.class, () -> getBuilder(Map.of("pizza", "yummy"))); + assertThat(e.getMessage(), equalTo("illegal kql query option [pizza]")); + + e = expectThrows(ZoneRulesException.class, () -> getBuilder(Map.of("time_zone", "aoeu"))); + assertThat(e.getMessage(), equalTo("Unknown time-zone ID: aoeu")); + } + + private static KqlQueryBuilder getBuilder(Map options) { + final Source source = new Source(1, 1, StringUtils.EMPTY); + final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", options); + return (KqlQueryBuilder) kqlQuery.asBuilder(); + } + + public void testToString() { + final Source source = new Source(1, 1, StringUtils.EMPTY); + final KqlQuery kqlQuery = new KqlQuery(source, "eggplant", Map.of()); + assertEquals("KqlQuery@1:2[eggplant]", kqlQuery.toString()); + } +} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java index 69767ce0b24f0..ba3e48e11928d 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultEndPointsIT.java @@ -51,6 +51,14 @@ public void tearDown() throws Exception { super.tearDown(); } + public void testGet() throws IOException { + var elserModel = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID); + assertDefaultElserConfig(elserModel); + + var e5Model = getModel(ElasticsearchInternalService.DEFAULT_E5_ID); + assertDefaultE5Config(e5Model); + } + @SuppressWarnings("unchecked") public void testInferDeploysDefaultElser() throws IOException { var model = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID); @@ -79,6 +87,7 @@ private static void assertDefaultElserConfig(Map modelConfig) { adaptiveAllocations, Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) ); + assertDefaultChunkingSettings(modelConfig); } @SuppressWarnings("unchecked") @@ -113,6 +122,17 @@ private static void assertDefaultE5Config(Map modelConfig) { adaptiveAllocations, Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 0, "max_number_of_allocations", 32)) ); + assertDefaultChunkingSettings(modelConfig); + } + + @SuppressWarnings("unchecked") + private static void assertDefaultChunkingSettings(Map modelConfig) { + var chunkingSettings = (Map) modelConfig.get("chunking_settings"); + assertThat( + modelConfig.toString(), + chunkingSettings, + Matchers.is(Map.of("strategy", "sentence", "max_chunk_size", 250, "sentence_overlap", 1)) + ); } public void testMultipleInferencesTriggeringDownloadAndDeploy() throws InterruptedException { diff --git a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java index 8cb37ad645358..c16271ed44083 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java +++ b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java @@ -135,6 +135,7 @@ public void testRerank() throws IOException { final String inferenceId = "mixed-cluster-rerank"; + cohereRerankServer.enqueue(new MockResponse().setResponseCode(200).setBody(rerankResponse())); put(inferenceId, rerankConfig(getUrl(cohereRerankServer)), TaskType.RERANK); assertRerank(inferenceId); diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java index 32969ffd1d112..0acbc148515bd 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/CohereServiceUpgradeIT.java @@ -201,6 +201,7 @@ public void testRerank() throws IOException { var testTaskType = TaskType.RERANK; if (isOldCluster()) { + cohereRerankServer.enqueue(new MockResponse().setResponseCode(200).setBody(rerankResponse())); put(oldClusterId, rerankConfig(getUrl(cohereRerankServer)), testTaskType); var configs = (List>) get(testTaskType, oldClusterId).get(old_cluster_endpoint_identifier); assertThat(configs, hasSize(1)); @@ -229,6 +230,7 @@ public void testRerank() throws IOException { assertRerank(oldClusterId); // New endpoint + cohereRerankServer.enqueue(new MockResponse().setResponseCode(200).setBody(rerankResponse())); put(upgradedClusterId, rerankConfig(getUrl(cohereRerankServer)), testTaskType); configs = (List>) get(upgradedClusterId).get("endpoints"); assertThat(configs, hasSize(1)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index c84b4314b9d1a..6d77663f49ece 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -18,7 +18,6 @@ import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.EmptySettingsConfiguration; -import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -51,6 +50,7 @@ import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; import java.util.HashMap; @@ -60,7 +60,6 @@ import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING; import static org.elasticsearch.inference.TaskType.TEXT_EMBEDDING; -import static org.elasticsearch.xpack.core.inference.action.InferenceAction.Request.DEFAULT_TIMEOUT; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -332,68 +331,39 @@ private EmbeddingRequestChunker.EmbeddingType getEmbeddingTypeFromTaskType(TaskT */ @Override public void checkModelConfig(Model model, ActionListener listener) { + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { if (model instanceof AlibabaCloudSearchEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) + var serviceSettings = embeddingsModel.getServiceSettings(); + + var updatedServiceSettings = new AlibabaCloudSearchEmbeddingsServiceSettings( + new AlibabaCloudSearchServiceSettings( + serviceSettings.getCommonSettings().modelId(), + serviceSettings.getCommonSettings().getHost(), + serviceSettings.getCommonSettings().getWorkspaceName(), + serviceSettings.getCommonSettings().getHttpSchema(), + serviceSettings.getCommonSettings().rateLimitSettings() + ), + SimilarityMeasure.DOT_PRODUCT, + embeddingSize, + serviceSettings.getMaxInputTokens() ); + + return new AlibabaCloudSearchEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - checkAlibabaCloudSearchServiceConfig(model, this, listener); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } - private AlibabaCloudSearchEmbeddingsModel updateModelWithEmbeddingDetails(AlibabaCloudSearchEmbeddingsModel model, int embeddingSize) { - AlibabaCloudSearchEmbeddingsServiceSettings serviceSettings = new AlibabaCloudSearchEmbeddingsServiceSettings( - new AlibabaCloudSearchServiceSettings( - model.getServiceSettings().getCommonSettings().modelId(), - model.getServiceSettings().getCommonSettings().getHost(), - model.getServiceSettings().getCommonSettings().getWorkspaceName(), - model.getServiceSettings().getCommonSettings().getHttpSchema(), - model.getServiceSettings().getCommonSettings().rateLimitSettings() - ), - SimilarityMeasure.DOT_PRODUCT, - embeddingSize, - model.getServiceSettings().getMaxInputTokens() - ); - - return new AlibabaCloudSearchEmbeddingsModel(model, serviceSettings); - } - @Override public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED; } - /** - * For other models except of text embedding - * check the model's service settings and task settings - * - * @param model The new model - * @param service The inferenceService - * @param listener The listener - */ - private void checkAlibabaCloudSearchServiceConfig(Model model, InferenceService service, ActionListener listener) { - String input = ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_INPUT; - String query = model.getTaskType().equals(TaskType.RERANK) ? ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_QUERY : null; - - service.infer( - model, - query, - List.of(input), - false, - Map.of(), - InputType.INGEST, - DEFAULT_TIMEOUT, - listener.delegateFailureAndWrap((delegate, r) -> { - listener.onResponse(model); - }) - ); - } - - private static final String ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_INPUT = "input"; - private static final String ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_QUERY = "query"; - public static class Configuration { public static InferenceServiceConfiguration get() { return configuration.getOrCompute(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index f9822c7ab4af9..a69b9d2c70405 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -49,6 +49,7 @@ import org.elasticsearch.xpack.inference.services.amazonbedrock.embeddings.AmazonBedrockEmbeddingsModel; import org.elasticsearch.xpack.inference.services.amazonbedrock.embeddings.AmazonBedrockEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.io.IOException; import java.util.EnumSet; @@ -303,49 +304,34 @@ public Set supportedStreamingTasks() { */ @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof AmazonBedrockEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } - private AmazonBedrockEmbeddingsModel updateModelWithEmbeddingDetails(AmazonBedrockEmbeddingsModel model, int embeddingSize) { - AmazonBedrockEmbeddingsServiceSettings serviceSettings = model.getServiceSettings(); - if (serviceSettings.dimensionsSetByUser() - && serviceSettings.dimensions() != null - && serviceSettings.dimensions() != embeddingSize) { - throw new ElasticsearchStatusException( - Strings.format( - "The retrieved embeddings size [%s] does not match the size specified in the settings [%s]. " - + "Please recreate the [%s] configuration with the correct dimensions", - embeddingSize, - serviceSettings.dimensions(), - model.getConfigurations().getInferenceEntityId() - ), - RestStatus.BAD_REQUEST + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof AmazonBedrockEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null + ? getProviderDefaultSimilarityMeasure(embeddingsModel.provider()) + : similarityFromModel; + + var updatedServiceSettings = new AmazonBedrockEmbeddingsServiceSettings( + serviceSettings.region(), + serviceSettings.modelId(), + serviceSettings.provider(), + embeddingSize, + serviceSettings.dimensionsSetByUser(), + serviceSettings.maxInputTokens(), + similarityToUse, + serviceSettings.rateLimitSettings() ); - } - - var similarityFromModel = serviceSettings.similarity(); - var similarityToUse = similarityFromModel == null ? getProviderDefaultSimilarityMeasure(model.provider()) : similarityFromModel; - - AmazonBedrockEmbeddingsServiceSettings settingsToUse = new AmazonBedrockEmbeddingsServiceSettings( - serviceSettings.region(), - serviceSettings.modelId(), - serviceSettings.provider(), - embeddingSize, - serviceSettings.dimensionsSetByUser(), - serviceSettings.maxInputTokens(), - similarityToUse, - serviceSettings.rateLimitSettings() - ); - return new AmazonBedrockEmbeddingsModel(model, settingsToUse); + return new AmazonBedrockEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } private static void checkProviderForTask(TaskType taskType, AmazonBedrockProvider provider) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java index 556b34b945c14..eba7353f2b12e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.inference.services.anthropic.completion.AnthropicChatCompletionModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; import java.util.HashMap; @@ -176,6 +177,12 @@ public AnthropicModel parsePersistedConfig(String inferenceEntityId, TaskType ta ); } + @Override + public void checkModelConfig(Model model, ActionListener listener) { + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + @Override public InferenceServiceConfiguration getConfiguration() { return Configuration.get(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index 6d36e5f6c8fe7..2f3a935cdf010 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -11,7 +11,6 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -46,6 +45,7 @@ import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsModel; import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; import java.util.HashMap; @@ -294,48 +294,32 @@ protected void doChunkedInfer( */ @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof AzureOpenAiEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } - private AzureOpenAiEmbeddingsModel updateModelWithEmbeddingDetails(AzureOpenAiEmbeddingsModel model, int embeddingSize) { - if (model.getServiceSettings().dimensionsSetByUser() - && model.getServiceSettings().dimensions() != null - && model.getServiceSettings().dimensions() != embeddingSize) { - throw new ElasticsearchStatusException( - Strings.format( - "The retrieved embeddings size [%s] does not match the size specified in the settings [%s]. " - + "Please recreate the [%s] configuration with the correct dimensions", - embeddingSize, - model.getServiceSettings().dimensions(), - model.getConfigurations().getInferenceEntityId() - ), - RestStatus.BAD_REQUEST + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof AzureOpenAiEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + + var updatedServiceSettings = new AzureOpenAiEmbeddingsServiceSettings( + serviceSettings.resourceName(), + serviceSettings.deploymentId(), + serviceSettings.apiVersion(), + embeddingSize, + serviceSettings.dimensionsSetByUser(), + serviceSettings.maxInputTokens(), + similarityToUse, + serviceSettings.rateLimitSettings() ); - } - - var similarityFromModel = model.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; - - AzureOpenAiEmbeddingsServiceSettings serviceSettings = new AzureOpenAiEmbeddingsServiceSettings( - model.getServiceSettings().resourceName(), - model.getServiceSettings().deploymentId(), - model.getServiceSettings().apiVersion(), - embeddingSize, - model.getServiceSettings().dimensionsSetByUser(), - model.getServiceSettings().maxInputTokens(), - similarityToUse, - model.getServiceSettings().rateLimitSettings() - ); - return new AzureOpenAiEmbeddingsModel(model, serviceSettings); + return new AzureOpenAiEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index de1d055e160da..cc67470686a02 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -45,6 +45,7 @@ import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; import java.util.HashMap; @@ -293,36 +294,35 @@ protected void doChunkedInfer( */ @Override public void checkModelConfig(Model model, ActionListener listener) { + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { if (model instanceof CohereEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? defaultSimilarity() : similarityFromModel; + + var updatedServiceSettings = new CohereEmbeddingsServiceSettings( + new CohereServiceSettings( + serviceSettings.getCommonSettings().uri(), + similarityToUse, + embeddingSize, + serviceSettings.getCommonSettings().maxInputTokens(), + serviceSettings.getCommonSettings().modelId(), + serviceSettings.getCommonSettings().rateLimitSettings() + ), + serviceSettings.getEmbeddingType() ); + + return new CohereEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - listener.onResponse(model); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } - private CohereEmbeddingsModel updateModelWithEmbeddingDetails(CohereEmbeddingsModel model, int embeddingSize) { - var userDefinedSimilarity = model.getServiceSettings().similarity(); - var similarityToUse = userDefinedSimilarity == null ? defaultSimilarity() : userDefinedSimilarity; - - CohereEmbeddingsServiceSettings serviceSettings = new CohereEmbeddingsServiceSettings( - new CohereServiceSettings( - model.getServiceSettings().getCommonSettings().uri(), - similarityToUse, - embeddingSize, - model.getServiceSettings().getCommonSettings().maxInputTokens(), - model.getServiceSettings().getCommonSettings().modelId(), - model.getServiceSettings().getCommonSettings().rateLimitSettings() - ), - model.getServiceSettings().getEmbeddingType() - ); - - return new CohereEmbeddingsModel(model, serviceSettings); - } - /** * Return the default similarity measure for the embedding type. * Cohere embeddings are normalized to unit vectors therefor Dot diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 718aeae979fe9..6d124906d65bd 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -862,6 +862,7 @@ public void updateModelsWithDynamicFields(List models, ActionListener> defaultsListener) { preferredModelVariantFn.accept(defaultsListener.delegateFailureAndWrap((delegate, preferredModelVariant) -> { if (PreferredModelVariant.LINUX_X86_OPTIMIZED.equals(preferredModelVariant)) { @@ -892,7 +893,7 @@ private List defaultConfigs(boolean useLinuxOptimizedModel) { new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32) ), ElserMlNodeTaskSettings.DEFAULT, - null // default chunking settings + ChunkingSettingsBuilder.DEFAULT_SETTINGS ); var defaultE5 = new MultilingualE5SmallModel( DEFAULT_E5_ID, @@ -904,7 +905,7 @@ private List defaultConfigs(boolean useLinuxOptimizedModel) { useLinuxOptimizedModel ? MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 : MULTILINGUAL_E5_SMALL_MODEL_ID, new AdaptiveAllocationsSettings(Boolean.TRUE, 0, 32) ), - null // default chunking settings + ChunkingSettingsBuilder.DEFAULT_SETTINGS ); return List.of(defaultElser, defaultE5); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java index a05b1a937d376..204593464a4ad 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java @@ -11,7 +11,6 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -45,6 +44,7 @@ import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankModel; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; import java.util.HashMap; @@ -181,15 +181,8 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof GoogleVertexAiEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } @Override @@ -240,34 +233,26 @@ protected void doChunkedInfer( } } - private GoogleVertexAiEmbeddingsModel updateModelWithEmbeddingDetails(GoogleVertexAiEmbeddingsModel model, int embeddingSize) { - if (model.getServiceSettings().dimensionsSetByUser() - && model.getServiceSettings().dimensions() != null - && model.getServiceSettings().dimensions() != embeddingSize) { - throw new ElasticsearchStatusException( - Strings.format( - "The retrieved embeddings size [%s] does not match the size specified in the settings [%s]. " - + "Please recreate the [%s] configuration with the correct dimensions", - embeddingSize, - model.getServiceSettings().dimensions(), - model.getConfigurations().getInferenceEntityId() - ), - RestStatus.BAD_REQUEST + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof GoogleVertexAiEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + + var updatedServiceSettings = new GoogleVertexAiEmbeddingsServiceSettings( + serviceSettings.location(), + serviceSettings.projectId(), + serviceSettings.modelId(), + serviceSettings.dimensionsSetByUser(), + serviceSettings.maxInputTokens(), + embeddingSize, + serviceSettings.similarity(), + serviceSettings.rateLimitSettings() ); - } - - GoogleVertexAiEmbeddingsServiceSettings serviceSettings = new GoogleVertexAiEmbeddingsServiceSettings( - model.getServiceSettings().location(), - model.getServiceSettings().projectId(), - model.getServiceSettings().modelId(), - model.getServiceSettings().dimensionsSetByUser(), - model.getServiceSettings().maxInputTokens(), - embeddingSize, - model.getServiceSettings().similarity(), - model.getServiceSettings().rateLimitSettings() - ); - return new GoogleVertexAiEmbeddingsModel(model, serviceSettings); + return new GoogleVertexAiEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } private static GoogleVertexAiModel createModelFromPersistent( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java index f4f4605c667c3..592900d117b39 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModel; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.EnumSet; import java.util.HashMap; @@ -228,35 +229,34 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void checkModelConfig(Model model, ActionListener listener) { + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { if (model instanceof IbmWatsonxEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + + var updatedServiceSettings = new IbmWatsonxEmbeddingsServiceSettings( + serviceSettings.modelId(), + serviceSettings.projectId(), + serviceSettings.url(), + serviceSettings.apiVersion(), + serviceSettings.maxInputTokens(), + embeddingSize, + similarityToUse, + serviceSettings.rateLimitSettings() ); + + return new IbmWatsonxEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - listener.onResponse(model); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } - private IbmWatsonxEmbeddingsModel updateModelWithEmbeddingDetails(IbmWatsonxEmbeddingsModel model, int embeddingSize) { - var similarityFromModel = model.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; - - IbmWatsonxEmbeddingsServiceSettings serviceSettings = new IbmWatsonxEmbeddingsServiceSettings( - model.getServiceSettings().modelId(), - model.getServiceSettings().projectId(), - model.getServiceSettings().url(), - model.getServiceSettings().apiVersion(), - model.getServiceSettings().maxInputTokens(), - embeddingSize, - similarityToUse, - model.getServiceSettings().rateLimitSettings() - ); - - return new IbmWatsonxEmbeddingsModel(model, serviceSettings); - } - @Override protected void doInfer( Model model, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java index aac111c22558e..b6d29ccab9a49 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.inference.services.alibabacloudsearch; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.bytes.BytesArray; @@ -22,6 +23,7 @@ import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -50,6 +52,7 @@ import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettingsTests; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettingsTests; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests; import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; @@ -325,6 +328,43 @@ public void doInfer( } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new AlibabaCloudSearchService(senderFactory, createWithEmptySettings(threadPool))) { + var model = OpenAiChatCompletionModelTests.createChatCompletionModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10) + ); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_UpdatesEmbeddingSizeAndSimilarity() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new AlibabaCloudSearchService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = AlibabaCloudSearchEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomFrom(TaskType.values()), + AlibabaCloudSearchEmbeddingsServiceSettingsTests.createRandom(), + AlibabaCloudSearchEmbeddingsTaskSettingsTests.createRandom(), + null + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + assertEquals(SimilarityMeasure.DOT_PRODUCT, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testChunkedInfer_TextEmbeddingChunkingSettingsSet() throws IOException { testChunkedInfer(TaskType.TEXT_EMBEDDING, ChunkingSettingsTests.createRandomChunkingSettings()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index e76fb10c96131..e583e50075ee7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.inference.services.amazonbedrock.embeddings.AmazonBedrockEmbeddingsModel; import org.elasticsearch.xpack.inference.services.amazonbedrock.embeddings.AmazonBedrockEmbeddingsModelTests; import org.elasticsearch.xpack.inference.services.amazonbedrock.embeddings.AmazonBedrockEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; @@ -72,6 +73,7 @@ import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockProviderCapabilities.getProviderDefaultSimilarityMeasure; import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockSecretSettingsTests.getAmazonBedrockSecretSettingsMap; import static org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionServiceSettingsTests.createChatCompletionRequestSettingsMap; import static org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionTaskSettingsTests.getChatCompletionTaskSettingsMap; @@ -1375,6 +1377,78 @@ public void testCheckModelConfig_ReturnsNewModelReference_AndDoesNotSendDimensio } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var sender = mock(Sender.class); + var factory = mock(HttpRequestSender.Factory.class); + when(factory.createSender()).thenReturn(sender); + + var amazonBedrockFactory = new AmazonBedrockMockRequestSender.Factory( + ServiceComponentsTests.createWithSettings(threadPool, Settings.EMPTY), + mockClusterServiceEmpty() + ); + + try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { + var model = AmazonBedrockChatCompletionModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomFrom(AmazonBedrockProvider.values()), + randomAlphaOfLength(10), + randomAlphaOfLength(10) + ); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var sender = mock(Sender.class); + var factory = mock(HttpRequestSender.Factory.class); + when(factory.createSender()).thenReturn(sender); + + var amazonBedrockFactory = new AmazonBedrockMockRequestSender.Factory( + ServiceComponentsTests.createWithSettings(threadPool, Settings.EMPTY), + mockClusterServiceEmpty() + ); + + try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var provider = randomFrom(AmazonBedrockProvider.values()); + var model = AmazonBedrockEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + provider, + randomNonNegativeInt(), + randomBoolean(), + randomNonNegativeInt(), + similarityMeasure, + RateLimitSettingsTests.createRandom(), + createRandomChunkingSettings(), + randomAlphaOfLength(10), + randomAlphaOfLength(10) + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null + ? getProviderDefaultSimilarityMeasure(provider) + : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testInfer_UnauthorizedResponse() throws IOException { var sender = mock(Sender.class); var factory = mock(HttpRequestSender.Factory.class); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index 40f8b7e0977e4..dc1970e26a3f8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -1194,6 +1194,53 @@ public void testCheckModelConfig_ReturnsNewModelReference_AndDoesNotSendDimensio } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new AzureOpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var model = AzureOpenAiCompletionModelTests.createModelWithRandomValues(); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new AzureOpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = AzureOpenAiEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + randomBoolean(), + randomNonNegativeInt(), + similarityMeasure, + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10) + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testInfer_UnauthorisedResponse() throws IOException, URISyntaxException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 725879e76efc1..30f3b344a268c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -1074,6 +1074,50 @@ public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosi } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { + var model = CohereCompletionModelTests.createModel(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = CohereEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + CohereEmbeddingsTaskSettings.EMPTY_SETTINGS, + randomNonNegativeInt(), + randomNonNegativeInt(), + randomAlphaOfLength(10), + randomFrom(CohereEmbeddingType.values()), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? CohereService.defaultSimilarity() : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testInfer_UnauthorisedResponse() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java index 906a825e49561..2aeba5fcbe209 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.http.MockWebServer; @@ -30,9 +31,11 @@ import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsModelTests; import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsTaskSettings; import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankModel; +import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankModelTests; import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankTaskSettings; import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; @@ -827,6 +830,37 @@ public void testParsePersistedConfig_CreatesAnEmbeddingsModelWhenChunkingSetting } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + try (var service = createGoogleVertexAiService()) { + var model = GoogleVertexAiRerankModelTests.createModel(randomAlphaOfLength(10), randomNonNegativeInt()); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + try (var service = createGoogleVertexAiService()) { + var embeddingSize = randomNonNegativeInt(); + var model = GoogleVertexAiEmbeddingsModelTests.createModel(randomAlphaOfLength(10), randomBoolean(), similarityMeasure); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + // testInfer tested via end-to-end notebook tests in AppEx repo @SuppressWarnings("checkstyle:LineLength") diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModelTests.java index 7836c5c15cfb1..5b016de7493f5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModelTests.java @@ -64,7 +64,7 @@ public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAre } public void testOverrideWith_SetsInputTypeToOverride_WhenFieldIsNullInModelTaskSettings_AndNullInRequestTaskSettings() { - var model = createModel("model", Boolean.FALSE, null); + var model = createModel("model", Boolean.FALSE, (InputType) null); var overriddenModel = GoogleVertexAiEmbeddingsModel.of(model, getTaskSettingsMap(null, null), InputType.SEARCH); var expectedModel = createModel("model", Boolean.FALSE, InputType.SEARCH); @@ -80,7 +80,7 @@ public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingStoredT } public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingRequestTaskSettings() { - var model = createModel("model", Boolean.FALSE, null); + var model = createModel("model", Boolean.FALSE, (InputType) null); var overriddenModel = GoogleVertexAiEmbeddingsModel.of(model, getTaskSettingsMap(null, InputType.CLUSTERING), InputType.SEARCH); var expectedModel = createModel("model", Boolean.FALSE, InputType.SEARCH); @@ -96,10 +96,10 @@ public void testOverrideWith_OverridesInputType_WithRequestTaskSettingsSearch_Wh } public void testOverrideWith_DoesNotSetInputType_FromRequest_IfInputTypeIsInvalid() { - var model = createModel("model", Boolean.FALSE, null); + var model = createModel("model", Boolean.FALSE, (InputType) null); var overriddenModel = GoogleVertexAiEmbeddingsModel.of(model, getTaskSettingsMap(null, null), InputType.UNSPECIFIED); - var expectedModel = createModel("model", Boolean.FALSE, null); + var expectedModel = createModel("model", Boolean.FALSE, (InputType) null); MatcherAssert.assertThat(overriddenModel, is(expectedModel)); } @@ -136,6 +136,31 @@ public static GoogleVertexAiEmbeddingsModel createModel( ); } + public static GoogleVertexAiEmbeddingsModel createModel( + String modelId, + @Nullable Boolean autoTruncate, + SimilarityMeasure similarityMeasure + ) { + return new GoogleVertexAiEmbeddingsModel( + "id", + TaskType.TEXT_EMBEDDING, + "service", + new GoogleVertexAiEmbeddingsServiceSettings( + randomAlphaOfLength(8), + randomAlphaOfLength(8), + modelId, + false, + null, + null, + similarityMeasure, + null + ), + new GoogleVertexAiEmbeddingsTaskSettings(autoTruncate, randomFrom(InputType.INGEST, InputType.SEARCH)), + null, + new GoogleVertexAiSecretSettings(new SecureString(randomAlphaOfLength(8).toCharArray())) + ); + } + public static GoogleVertexAiEmbeddingsModel createModel(String modelId, @Nullable Boolean autoTruncate, @Nullable InputType inputType) { return new GoogleVertexAiEmbeddingsModel( "id", diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java index f7f37c5bcd15f..1261e3834437b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModel; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.After; @@ -930,6 +931,56 @@ public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosi } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new IbmWatsonxServiceWithoutAuth(senderFactory, createWithEmptySettings(threadPool))) { + var model = OpenAiChatCompletionModelTests.createChatCompletionModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10) + ); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new IbmWatsonxServiceWithoutAuth(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = IbmWatsonxEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + URI.create(randomAlphaOfLength(10)), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testGetConfiguration() throws Exception { try (var service = createIbmWatsonxService()) { String content = XContentHelper.stripWhitespace(""" diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java index 5dff9126b6be4..e2817665d8f79 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -26,6 +27,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.kql.parser.KqlParser; import org.elasticsearch.xpack.kql.parser.KqlParsingContext; +import org.elasticsearch.xpack.kql.parser.KqlParsingException; import java.io.IOException; import java.time.ZoneId; @@ -37,9 +39,9 @@ public class KqlQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "kql"; public static final ParseField QUERY_FIELD = new ParseField("query"); - private static final ParseField CASE_INSENSITIVE_FIELD = new ParseField("case_insensitive"); - private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); - private static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); + public static final ParseField CASE_INSENSITIVE_FIELD = new ParseField("case_insensitive"); + public static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + public static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); private static final Logger log = LogManager.getLogger(KqlQueryBuilder.class); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, a -> { @@ -151,12 +153,16 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException { - KqlParser parser = new KqlParser(); - QueryBuilder rewrittenQuery = parser.parseKqlQuery(query, createKqlParserContext(context)); + try { + KqlParser parser = new KqlParser(); + QueryBuilder rewrittenQuery = parser.parseKqlQuery(query, createKqlParserContext(context)); - log.trace(() -> Strings.format("KQL query %s translated to Query DSL: %s", query, Strings.toString(rewrittenQuery))); + log.trace(() -> Strings.format("KQL query %s translated to Query DSL: %s", query, Strings.toString(rewrittenQuery))); - return rewrittenQuery; + return rewrittenQuery; + } catch (KqlParsingException e) { + throw new QueryShardException(context, "Failed to parse KQL query [{}]", e, query); + } } @Override diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java index f529b9fa1db96..99acbec04551e 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeCustomSettingsIT.java @@ -7,9 +7,11 @@ package org.elasticsearch.xpack.logsdb; +import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.junit.Before; @@ -112,8 +114,11 @@ public void testConfigureStoredSourceBeforeIndexCreation() throws IOException { }"""; assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping)); - assertOK(createDataStream(client, "logs-custom-dev")); - + Request request = new Request("PUT", "_data_stream/logs-custom-dev"); + if (SourceFieldMapper.onOrAfterDeprecateModeVersion(minimumIndexVersion())) { + request.setOptions(expectVersionSpecificWarnings(v -> v.current(SourceFieldMapper.DEPRECATION_WARNING))); + } + assertOK(client.performRequest(request)); var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); assertThat(sourceMode, equalTo("stored")); @@ -182,7 +187,11 @@ public void testConfigureStoredSourceWhenIndexIsCreated() throws IOException { }"""; assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping)); - assertOK(createDataStream(client, "logs-custom-dev")); + Request request = new Request("PUT", "_data_stream/logs-custom-dev"); + if (SourceFieldMapper.onOrAfterDeprecateModeVersion(minimumIndexVersion())) { + request.setOptions(expectVersionSpecificWarnings(v -> v.current(SourceFieldMapper.DEPRECATION_WARNING))); + } + assertOK(client.performRequest(request)); var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java index cc7f5bdb33871..0990592cef5e3 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; @@ -35,6 +36,11 @@ protected static Response putComponentTemplate(final RestClient client, final St throws IOException { final Request request = new Request("PUT", "/_component_template/" + componentTemplate); request.setJsonEntity(contends); + if (isSyntheticSourceConfiguredInTemplate(contends) && SourceFieldMapper.onOrAfterDeprecateModeVersion(minimumIndexVersion())) { + request.setOptions( + expectVersionSpecificWarnings((VersionSensitiveWarningsHandler v) -> v.current(SourceFieldMapper.DEPRECATION_WARNING)) + ); + } return client.performRequest(request); } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index 8930ff23fb3b0..e411f2f3f314d 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -181,7 +181,7 @@ protected static void waitForLogs(RestClient client) throws Exception { } public void testMatchAllQuery() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + int numberOfDocuments = ESTestCase.randomIntBetween(20, 100); final List documents = generateDocuments(numberOfDocuments); indexDocuments(documents); @@ -199,7 +199,7 @@ public void testMatchAllQuery() throws IOException { } public void testTermsQuery() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + int numberOfDocuments = ESTestCase.randomIntBetween(20, 100); final List documents = generateDocuments(numberOfDocuments); indexDocuments(documents); @@ -217,7 +217,7 @@ public void testTermsQuery() throws IOException { } public void testHistogramAggregation() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + int numberOfDocuments = ESTestCase.randomIntBetween(20, 100); final List documents = generateDocuments(numberOfDocuments); indexDocuments(documents); @@ -235,7 +235,7 @@ public void testHistogramAggregation() throws IOException { } public void testTermsAggregation() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + int numberOfDocuments = ESTestCase.randomIntBetween(20, 100); final List documents = generateDocuments(numberOfDocuments); indexDocuments(documents); @@ -253,7 +253,7 @@ public void testTermsAggregation() throws IOException { } public void testDateHistogramAggregation() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + int numberOfDocuments = ESTestCase.randomIntBetween(20, 100); final List documents = generateDocuments(numberOfDocuments); indexDocuments(documents); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java index 1f5d26eaedf34..d6cdb9f761b31 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java @@ -81,10 +81,12 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); } { String mapping; - if (randomBoolean()) { + boolean withSourceMode = randomBoolean(); + if (withSourceMode) { mapping = """ { "_doc": { @@ -115,6 +117,9 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); + if (withSourceMode) { + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); + } } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java index 447bca4f4e688..94fbde69e29c7 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.ml.integration; -import org.apache.lucene.util.Constants; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -222,8 +221,6 @@ public void testMemoryStatus() { } public void testOverflowToDisk() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/44609", Constants.WINDOWS); - Detector.Builder detector = new Detector.Builder("mean", "value"); detector.setByFieldName("clientIP"); diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 2b0d1ec536fa6..3a1ba435b8f1f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -10,8 +10,6 @@ template: sort: field: [ "resource.attributes.host.name", "@timestamp" ] mappings: - _source: - mode: synthetic properties: trace_id: type: keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index b2d30c7f85cc4..9edbe5622b3f1 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin otel-data. This must be increased whenever an existing template is # changed, in order for it to be updated on Elasticsearch upgrade. -version: 6 +version: 7 component-templates: - otel@mappings diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java index 7d8a474453c4c..71e8dcbff4ee6 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java @@ -54,7 +54,7 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry { // version 11: Added 'profiling.agent.protocol' keyword mapping to profiling-hosts // version 12: Added 'profiling.agent.env_https_proxy' keyword mapping to profiling-hosts // version 13: Added 'container.id' keyword mapping to profiling-events - public static final int INDEX_TEMPLATE_VERSION = 13; + public static final int INDEX_TEMPLATE_VERSION = 14; // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index public static final int PROFILING_EVENTS_VERSION = 5; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java index 0ee291af29ae1..a44764dab2a38 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java @@ -113,7 +113,7 @@ public T[] toArray(T[] a) { @Override public boolean add(Attribute e) { - return delegate.put(e, PRESENT) != null; + return delegate.put(e, PRESENT) == null; } @Override diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java index b7674a2d60bff..0ab3e99e1efc9 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java @@ -64,12 +64,12 @@ public void testNextExecutionTimeSchedule() { SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy( "id", "name", - "0 1 2 3 4 ? 2099", + "0 1 2 3 4 ? 2049", "repo", Collections.emptyMap(), SnapshotRetentionConfiguration.EMPTY ); - assertThat(p.calculateNextExecution(-1, Clock.systemUTC()), equalTo(4078864860000L)); + assertThat(p.calculateNextExecution(-1, Clock.systemUTC()), equalTo(2501028060000L)); } public void testNextExecutionTimeInterval() { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 72c7c51655378..f7dd979540afa 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 121} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 122} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/11_timezoned_schedules.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/11_timezoned_schedules.yml new file mode 100644 index 0000000000000..0371443367603 --- /dev/null +++ b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/11_timezoned_schedules.yml @@ -0,0 +1,121 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +"Test put watch api with timezone": + - do: + watcher.put_watch: + id: "my_watch" + body: > + { + "trigger": { + "schedule": { + "timezone": "America/Los_Angeles", + "hourly": { + "minute": [ 0, 5 ] + } + } + }, + "input": { + "simple": { + "payload": { + "send": "yes" + } + } + }, + "condition": { + "always": {} + }, + "actions": { + "test_index": { + "index": { + "index": "test" + } + } + } + } + - match: { _id: "my_watch" } + - do: + watcher.get_watch: + id: "my_watch" + - match: { watch.trigger.schedule.timezone: "America/Los_Angeles" } + +--- +"Test put watch api without timezone": + - do: + watcher.put_watch: + id: "my_watch" + body: > + { + "trigger": { + "schedule": { + "hourly": { + "minute": [ 0, 5 ] + } + } + }, + "input": { + "simple": { + "payload": { + "send": "yes" + } + } + }, + "condition": { + "always": {} + }, + "actions": { + "test_index": { + "index": { + "index": "test" + } + } + } + } + - match: { _id: "my_watch" } + - do: + watcher.get_watch: + id: "my_watch" + - is_false: watch.trigger.schedule.timezone + +--- +"Reject put watch with invalid timezone": + - do: + watcher.put_watch: + id: "my_watch" + body: > + { + "trigger": { + "schedule": { + "timezone": "Pangea/Tethys", + "hourly": { + "minute": [ 0, 5 ] + } + } + }, + "input": { + "simple": { + "payload": { + "send": "yes" + } + } + }, + "condition": { + "always": {} + }, + "actions": { + "test_index": { + "index": { + "index": "test" + } + } + } + } + catch: bad_request + - match: { error.type: "parse_exception" } + - match: { error.reason: "could not parse schedule. invalid timezone [Pangea/Tethys]" } + - match: { error.caused_by.type: "zone_rules_exception" } + - match: { error.caused_by.reason: "Unknown time-zone ID: Pangea/Tethys" } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/CronnableSchedule.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/CronnableSchedule.java index 63e9dae88de41..0db99af9b3fc2 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/CronnableSchedule.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/CronnableSchedule.java @@ -8,6 +8,7 @@ import org.elasticsearch.xpack.core.scheduler.Cron; +import java.time.ZoneId; import java.util.Arrays; import java.util.Comparator; import java.util.Objects; @@ -17,6 +18,7 @@ public abstract class CronnableSchedule implements Schedule { private static final Comparator CRON_COMPARATOR = Comparator.comparing(Cron::expression); protected final Cron[] crons; + private ZoneId timeZone; CronnableSchedule(String... expressions) { this(crons(expressions)); @@ -28,6 +30,17 @@ private CronnableSchedule(Cron... crons) { Arrays.sort(crons, CRON_COMPARATOR); } + protected void setTimeZone(ZoneId timeZone) { + this.timeZone = timeZone; + for (Cron cron : crons) { + cron.setTimeZone(timeZone); + } + } + + public ZoneId getTimeZone() { + return timeZone; + } + @Override public long nextScheduledTimeAfter(long startTime, long time) { assert time >= startTime; @@ -45,21 +58,22 @@ public Cron[] crons() { return crons; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CronnableSchedule that = (CronnableSchedule) o; + return Objects.deepEquals(crons, that.crons) && Objects.equals(timeZone, that.timeZone); + } + @Override public int hashCode() { - return Objects.hash((Object[]) crons); + return Objects.hash(Arrays.hashCode(crons), timeZone); } @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final CronnableSchedule other = (CronnableSchedule) obj; - return Objects.deepEquals(this.crons, other.crons); + public String toString() { + return "CronnableSchedule{" + "crons=" + Arrays.toString(crons) + ", timeZone=" + timeZone + '}'; } static Cron[] crons(String... expressions) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistry.java index 31cf46f8abaac..5d2259db71f77 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistry.java @@ -8,8 +8,11 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.watcher.trigger.schedule.support.TimezoneUtils; import java.io.IOException; +import java.time.DateTimeException; +import java.time.ZoneId; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -29,9 +32,15 @@ public Schedule parse(String context, XContentParser parser) throws IOException String type = null; XContentParser.Token token; Schedule schedule = null; + ZoneId timeZone = null; // Default to UTC while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { - type = parser.currentName(); + var fieldName = parser.currentName(); + if (fieldName.equals(ScheduleTrigger.TIMEZONE_FIELD)) { + timeZone = parseTimezone(parser); + } else { + type = parser.currentName(); + } } else if (type != null) { schedule = parse(context, type, parser); } else { @@ -44,9 +53,38 @@ public Schedule parse(String context, XContentParser parser) throws IOException if (schedule == null) { throw new ElasticsearchParseException("could not parse schedule. expected a schedule type field, but no fields were found"); } + + if (timeZone != null && schedule instanceof CronnableSchedule cronnableSchedule) { + cronnableSchedule.setTimeZone(timeZone); + } else if (timeZone != null) { + throw new ElasticsearchParseException( + "could not parse schedule. Timezone is not supported for schedule type [{}]", + schedule.type() + ); + } + return schedule; } + private static ZoneId parseTimezone(XContentParser parser) throws IOException { + ZoneId timeZone; + XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.VALUE_STRING) { + String text = parser.text(); + try { + timeZone = TimezoneUtils.parse(text); + } catch (DateTimeException e) { + throw new ElasticsearchParseException("could not parse schedule. invalid timezone [{}]", e, text); + } + } else { + throw new ElasticsearchParseException( + "could not parse schedule. expected a string value for timezone, but found [{}] instead", + token + ); + } + return timeZone; + } + public Schedule parse(String context, String type, XContentParser parser) throws IOException { Schedule.Parser scheduleParser = parsers.get(type); if (scheduleParser == null) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTrigger.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTrigger.java index 4a67841e6c88e..cc6ec8f5aaa57 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTrigger.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTrigger.java @@ -14,6 +14,7 @@ public class ScheduleTrigger implements Trigger { public static final String TYPE = "schedule"; + public static final String TIMEZONE_FIELD = "timezone"; private final Schedule schedule; @@ -49,7 +50,13 @@ public int hashCode() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject().field(schedule.type(), schedule, params).endObject(); + builder.startObject(); + if (schedule instanceof CronnableSchedule cronnableSchedule && cronnableSchedule.getTimeZone() != null) { + builder.field(TIMEZONE_FIELD, cronnableSchedule.getTimeZone().getId()); + } + + builder.field(schedule.type(), schedule, params); + return builder.endObject(); } public static Builder builder(Schedule schedule) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/support/TimezoneUtils.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/support/TimezoneUtils.java new file mode 100644 index 0000000000000..c77fdda803bec --- /dev/null +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/support/TimezoneUtils.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.watcher.trigger.schedule.support; + +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.Locale; +import java.util.Map; + +import static java.util.stream.Collectors.toMap; + +/** + * Utility class for dealing with Timezone related operations. + */ +public class TimezoneUtils { + + private static final Map caseInsensitiveTZLookup; + + static { + caseInsensitiveTZLookup = ZoneId.getAvailableZoneIds() + .stream() + .collect(toMap(zoneId -> zoneId.toLowerCase(Locale.ROOT), ZoneId::of)); + } + + /** + * Parses a timezone string into a {@link ZoneId} object. The timezone string can be a valid timezone ID, or a + * timezone offset string and is case-insensitive. + * + * @param timezoneString The timezone string to parse + * @return The parsed {@link ZoneId} object + * @throws DateTimeException If the timezone string is not a valid timezone ID or offset + */ + public static ZoneId parse(String timezoneString) throws DateTimeException { + try { + return ZoneId.of(timezoneString); + } catch (DateTimeException e) { + ZoneId timeZone = caseInsensitiveTZLookup.get(timezoneString.toLowerCase(Locale.ROOT)); + if (timeZone != null) { + return timeZone; + } + try { + return ZoneId.of(timezoneString.toUpperCase(Locale.ROOT)); + } catch (DateTimeException ignored) { + // ignore + } + throw e; + } + } + +} diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistryTests.java index 7fc4739c342f1..aa39701d207c3 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleRegistryTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; +import java.time.ZoneId; import java.util.HashSet; import java.util.Set; @@ -49,15 +50,23 @@ public void testParserInterval() throws Exception { } public void testParseCron() throws Exception { - Object cron = randomBoolean() ? Schedules.cron("* 0/5 * * * ?") : Schedules.cron("* 0/2 * * * ?", "* 0/3 * * * ?", "* 0/5 * * * ?"); - XContentBuilder builder = jsonBuilder().startObject().field(CronSchedule.TYPE, cron).endObject(); + var cron = randomBoolean() ? Schedules.cron("* 0/5 * * * ?") : Schedules.cron("* 0/2 * * * ?", "* 0/3 * * * ?", "* 0/5 * * * ?"); + ZoneId timeZone = null; + XContentBuilder builder = jsonBuilder().startObject().field(CronSchedule.TYPE, cron); + if (randomBoolean()) { + timeZone = randomTimeZone().toZoneId(); + cron.setTimeZone(timeZone); + builder.field(ScheduleTrigger.TIMEZONE_FIELD, timeZone.getId()); + } + builder.endObject(); BytesReference bytes = BytesReference.bytes(builder); XContentParser parser = createParser(JsonXContent.jsonXContent, bytes); parser.nextToken(); - Schedule schedule = registry.parse("ctx", parser); + CronnableSchedule schedule = (CronnableSchedule) registry.parse("ctx", parser); assertThat(schedule, notNullValue()); assertThat(schedule, instanceOf(CronSchedule.class)); assertThat(schedule, is(cron)); + assertThat(schedule.getTimeZone(), equalTo(timeZone)); } public void testParseHourly() throws Exception { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/support/TimezoneUtilsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/support/TimezoneUtilsTests.java new file mode 100644 index 0000000000000..aa797ec610eca --- /dev/null +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/support/TimezoneUtilsTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.watcher.trigger.schedule.support; + +import org.elasticsearch.test.ESTestCase; + +import java.time.ZoneId; +import java.util.Locale; + +import static org.hamcrest.Matchers.equalTo; + +public class TimezoneUtilsTests extends ESTestCase { + + public void testExpectedFormatParsing() { + assertThat(TimezoneUtils.parse("Europe/London").getId(), equalTo("Europe/London")); + assertThat(TimezoneUtils.parse("+1").getId(), equalTo("+01:00")); + assertThat(TimezoneUtils.parse("GMT+01:00").getId(), equalTo("GMT+01:00")); + } + + public void testParsingIsCaseInsensitive() { + ZoneId timeZone = randomTimeZone().toZoneId(); + assertThat(TimezoneUtils.parse(timeZone.getId()), equalTo(timeZone)); + assertThat(TimezoneUtils.parse(timeZone.getId().toLowerCase(Locale.ROOT)), equalTo(timeZone)); + assertThat(TimezoneUtils.parse(timeZone.getId().toUpperCase(Locale.ROOT)), equalTo(timeZone)); + } + + public void testParsingOffsets() { + ZoneId timeZone = ZoneId.of("GMT+01:00"); + assertThat(TimezoneUtils.parse("GMT+01:00"), equalTo(timeZone)); + assertThat(TimezoneUtils.parse("gmt+01:00"), equalTo(timeZone)); + assertThat(TimezoneUtils.parse("GMT+1"), equalTo(timeZone)); + + assertThat(TimezoneUtils.parse("+1"), equalTo(ZoneId.of("+01:00"))); + } +}